def run_experiment(experiment_config: Dict, save_weights: bool, gpu_ind: int, use_wandb: bool = True):
"""Run a training experiment."""
print(f'Running experiment with config {experiment_config} on GPU {gpu_ind}')
datasets_module = importlib.import_module('text_recognizer.datasets')
dataset_class_ = getattr(datasets_module, experiment_config['dataset'])
dataset_args = experiment_config.get('dataset_args', {})
dataset = dataset_class_(**dataset_args)
dataset.load_or_generate_data()
print(dataset)
models_module = importlib.import_module('text_recognizer.models')
model_class_ = getattr(models_module, experiment_config['model'])
networks_module = importlib.import_module('text_recognizer.networks')
network_fn_ = getattr(networks_module, experiment_config['network'])
network_args = experiment_config.get('network_args', {})
model = model_class_(dataset_cls=dataset_class_, network_fn=network_fn_, dataset_args=dataset_args,
network_args=network_args)
print(model)
experiment_config['train_args'] = {**DEFAULT_TRAIN_ARGS, **experiment_config.get('train_args', {})}
experiment_config['experiment_group'] = experiment_config.get('experiment_group', None)
experiment_config['gpu_ind'] = gpu_ind
train_model(model, dataset, epochs=experiment_config['train_args']['epochs'],
batch_size=experiment_config['train_args']['batch_size'], use_wandb=use_wandb)
score = model.evaluate(dataset.x_test, dataset.y_test)
print(f'Test evaluation: {score}')
if save_weights:
model.save_weights()
def run_experiment(experiment_config: Dict, save_weights: bool, gpu_ind: int, use_wandb: bool=True):
"""
experiment_config is of the form
{
"dataset": "EmnistLinesDataset",
"dataset_args": {
"max_overlap": 0.4
},
"model": "LineModel",
"network": "line_cnn_sliding_window",
"network_args": {
"window_width": 14,
"window_stride": 7
},
"train_args": {
"batch_size": 128,
"epochs": 10
}
}
save_weights: if True, will save the final model weights to a canonical location (see Model in models/base.py)
gpu_ind: integer specifying which gpu to use
"""
print(f'Running experiment with config {experiment_config} on GPU {gpu_ind}')
datasets_module = importlib.import_module('text_recognizer.datasets')
dataset_class_ = getattr(datasets_module, experiment_config['dataset'])
dataset_args = experiment_config.get('dataset_args', {})
dataset = dataset_class_(**dataset_args)
dataset.load_or_generate_data()
print(dataset)
models_module = importlib.import_module('text_recognizer.models')
model_class_ = getattr(models_module, experiment_config['model'])
networks_module = importlib.import_module('text_recognizer.networks')
network_fn_ = getattr(networks_module, experiment_config['network'])
network_args = experiment_config.get('network_args', {})
model = model_class_(dataset_cls=dataset_class_, network_fn=network_fn_, dataset_args=dataset_args, network_args=network_args)
print(model)
experiment_config['train_args'] = {**DEFAULT_TRAIN_ARGS, **experiment_config.get('train_args', {})}
experiment_config['experiment_group'] = experiment_config.get('experiment_group', None)
experiment_config['gpu_ind'] = gpu_ind
train_model(
model,
dataset,
epochs=experiment_config['train_args']['epochs'],
batch_size=experiment_config['train_args']['batch_size'],
gpu_ind=gpu_ind,
use_wandb=use_wandb
)
score = model.evaluate(dataset.x_test, dataset.y_test)
print(f'Test evaluation: {score}')
if save_weights:
model.save_weights()
def run_experiment(experiment_config: Dict,
save_weights: bool,
gpu_ind: int,
use_wandb: bool = True):
"""
Run a training experiment.
Parameters
----------
experiment_config (dict)
Of the form
{
"dataset": "EmnistLinesDataset",
"dataset_args": {
"max_overlap": 0.4,
"subsample_fraction": 0.2
},
"model": "LineModel",
"network": "line_cnn_all_conv",
"network_args": {
"window_width": 14,
"window_stride": 7
},
"train_args": {
"batch_size": 128,
"epochs": 10
}
}
save_weights (bool)
If True, will save the final model weights to a canonical location (see Model in models/base.py)
gpu_ind (int)
specifies which gpu to use (or -1 for first available)
use_wandb (bool)
sync training run to wandb
"""
print(
f"Running experiment with config {experiment_config} on GPU {gpu_ind}")
datasets_module = importlib.import_module("text_recognizer.datasets")
dataset_class_ = getattr(datasets_module, experiment_config["dataset"])
dataset_args = experiment_config.get("dataset_args", {})
dataset = dataset_class_(**dataset_args)
dataset.load_or_generate_data()
print(dataset)
models_module = importlib.import_module("text_recognizer.models")
model_class_ = getattr(models_module, experiment_config["model"])
networks_module = importlib.import_module("text_recognizer.networks")
network_fn_ = getattr(networks_module, experiment_config["network"])
network_args = experiment_config.get("network_args", {})
model = model_class_(
dataset_cls=dataset_class_,
network_fn=network_fn_,
dataset_args=dataset_args,
network_args=network_args,
)
print(model)
experiment_config["train_args"] = {
**DEFAULT_TRAIN_ARGS,
**experiment_config.get("train_args", {}),
}
experiment_config["experiment_group"] = experiment_config.get(
"experiment_group", None)
experiment_config["gpu_ind"] = gpu_ind
# Hide lines below until Lab 4
if use_wandb:
wandb.init(config=experiment_config)
# Hide lines above until Lab 4
train_model(
model,
dataset,
epochs=experiment_config["train_args"]["epochs"],
batch_size=experiment_config["train_args"]["batch_size"],
use_wandb=use_wandb,
)
score = model.evaluate(dataset.x_test, dataset.y_test)
print(f"Test evaluation: {score}")
# Hide lines below until Lab 4
if use_wandb:
wandb.log({"test_metric": score})
# Hide lines above until Lab 4
if save_weights:
model.save_weights()
def run_experiment(experiment_config: Dict,
save_weights: bool,
gpu_ind: int,
use_wandb: bool = True):
"""
Run a training experiment.
Parameters
----------
experiment_config (dict)
Of the form
{
"dataset": "EmnistLinesDataset",
"dataset_args": {
"max_overlap": 0.4,
"subsample_fraction": 0.2
},
"model": "LineModel",
"network": "line_cnn_all_conv",
"network_args": {
"window_width": 14,
"window_stride": 7
},
"train_args": {
"batch_size": 128,
"epochs": 10
}
}
save_weights (bool)
If True, will save the final model weights to a canonical location (see Model in models/base.py)
gpu_ind (int)
specifies which gpu to use (or -1 for first available)
use_wandb (bool)
sync training run to wandb
"""
print(
f'Running experiment with config {experiment_config} on GPU {gpu_ind}')
tf.enable_eager_execution()
datasets_module = importlib.import_module('mss.datasets')
dataset_class_ = getattr(datasets_module, experiment_config['dataset'])
dataset_args = {
**DEFAULT_DATASET_ARGS,
**experiment_config.get('dataset_args', {})
}
models_module = importlib.import_module('mss.models')
model_class_ = getattr(models_module, experiment_config['model'])
model_args = {
**DEFAULT_MODEL_ARGS,
**experiment_config.get('model_args', {})
}
networks_module = importlib.import_module('mss.networks')
network_fn_ = getattr(networks_module, experiment_config['network'])
network_args = experiment_config.get('network_args', {})
train_args = {
**DEFAULT_TRAIN_ARGS,
**experiment_config.get('train_args', {})
}
model = model_class_(dataset_cls=dataset_class_,
network_fn=network_fn_,
dataset_args=dataset_args,
network_args=network_args,
train_args=train_args,
model_args=model_args)
experiment_config['experiment_group'] = experiment_config.get(
'experiment_group', None)
experiment_config['gpu_ind'] = gpu_ind
# Hide lines below until Lab 4
#if use_wandb:
# wandb.init()
# wandb.config.update(experiment_config)
# Hide lines above until Lab 4
train_model(model,
epochs=train_args['epochs'],
batch_size=train_args['batch_size'],
gpu_ind=gpu_ind,
use_wandb=use_wandb)
if save_weights:
model.save_weights()
score = model.evaluate()
print(f'Test evaluation: {score}')
def run_experiment(experiment_config: Dict,
save_weights: bool,
gpu_ind: int,
use_wandb: bool = True):
print(
f'Running experiment with config {experiment_config}, on GPU {gpu_ind}'
)
datasets_module = importlib.import_module('architecture.datasets')
dataset_class_ = getattr(datasets_module, experiment_config['dataset'])
dataset_args = experiment_config.get('dataset_args', {})
dataset = dataset_class_(**dataset_args)
dataset.load_or_generate_data()
print(dataset)
models_module = importlib.import_module('architecture.models')
model_class_ = getattr(models_module, experiment_config['model'])
networks_module = importlib.import_module('architecture.networks')
network_fn_ = getattr(networks_module, experiment_config['network'])
network_args = experiment_config.get('network_args', {})
opt_args_ = experiment_config.get('opt_args', DEFAULT_OPT_ARGS)
model = model_class_(dataset_cls=dataset_class_,
network_fn=network_fn_,
dataset_args=dataset_args,
network_args=network_args,
opt_args=opt_args_)
experiment_config["train_args"] = {
**DEFAULT_TRAIN_ARGS,
**experiment_config.get("train_args", {}),
}
experiment_config["opt_args"] = {
**DEFAULT_OPT_ARGS,
**experiment_config.get("opt_args", {}),
}
experiment_config["experiment_group"] = experiment_config.get(
"experiment_group", None)
experiment_config["gpu_ind"] = gpu_ind
if use_wandb:
dataset_name = {
'AlzheimerT2SmallDataset': 't2mini',
'AlzheimerT2StarSmallDataset': 't2starmini',
'AlzheimerT2StarFullDataset': 't2starfull',
'AlzheimerMPRageDeep': 'mprage_deep',
'AlzheimerMPRageNoDeep': 'mprage_nodeep',
}
tags = []
tags.append('-'.join(list(dataset.mapping.values())).lower())
if dataset.num_classes > 2:
tags.append('binary')
else:
tags.append('multiclass')
wandb.init(
project='alzheimer-dl',
config=experiment_config,
name='{model} {dataset} {epochs}ep {batch_size}bs'.format(
model=('multi ' if dataset.num_classes > 2 else 'bin ') +
experiment_config['network'],
dataset=dataset_name[experiment_config['dataset']],
epochs=experiment_config["train_args"]["epochs"],
batch_size=experiment_config["train_args"]["batch_size"],
tags=tags))
with tf.device('/GPU:0'):
train_model(
model,
dataset,
epochs=experiment_config["train_args"]["epochs"],
batch_size=experiment_config["train_args"]["batch_size"],
use_wandb=use_wandb,
)
if use_wandb:
classes = list(dataset.mapping.values())
cm_val = plot_confusion_matrix(
model, dataset.X_val, dataset.y_val, classes,
experiment_config["train_args"]["batch_size"])
wandb.log({"confusion_matrix - validation data": cm_val})
cm_test = plot_confusion_matrix(
model, dataset.X_test, dataset.y_test, classes,
experiment_config["train_args"]["batch_size"])
wandb.log({"confusion_matrix - test": cm_test})
if save_weights:
model.save_weights()
def run_experiment(experiment_config: Dict, save_weights: bool, gpu_ind: int, use_experiment_manager: bool = True):
"""
Run a training experiment.
Parameters
----------
experiment_config (dict)
Of the form
{
"dataset": "Dataset",
"dataset_args": {
"data_arg": 0.4,
},
"model": "Model",
"network": "neural_net",
"network_args": {
"hidden_size": 256,
},
"train_args": {
"batch_size": 128,
"epochs": 10
}
}
save_weights (bool)
If True, will save the final model weights to a canonical location (see Model in models/base.py)
gpu_ind (int)
specifies which gpu to use
use_experiment_manager (bool)
sync training run to wandb, tensorboard, etc.
"""
print(f"Running experiment with config {experiment_config} on GPU {gpu_ind}")
datasets_module = importlib.import_module("model_core.datasets")
dataset_class_ = getattr(datasets_module, experiment_config["dataset"])
dataset_args = experiment_config.get("dataset_args", {})
dataset = dataset_class_(**dataset_args)
dataset.load_or_generate_data()
print(dataset)
models_module = importlib.import_module("model_core.models")
model_class_ = getattr(models_module, experiment_config["model"])
networks_module = importlib.import_module("model_core.networks")
network_fn_ = getattr(networks_module, experiment_config["network"])
network_args = experiment_config.get("network_args", {})
model = model_class_(
dataset_cls=dataset_class_, network_fn=network_fn_, dataset_args=dataset_args, network_args=network_args,
)
print(model)
experiment_config["train_args"] = {
**DEFAULT_TRAIN_ARGS,
**experiment_config.get("train_args", {}),
}
experiment_config["experiment_group"] = experiment_config.get("experiment_group", None)
experiment_config["gpu_ind"] = gpu_ind
train_model(
model,
dataset,
epochs=experiment_config["train_args"]["epochs"],
batch_size=experiment_config["train_args"]["batch_size"],
use_experiment_manager=use_experiment_manager,
)
score = model.evaluate(dataset.x_test, dataset.y_test)
print(f"Test evaluation: {score}")
if save_weights:
model.save_weights()
def run_experiment(experiment_config: Dict, save_weights: bool, gpu_ind: int, use_wandb: bool = True):
"""
Run a training experiment.
For defaults see: base.fit, network,
Parameters
----------
experiment_config (dict)
Of the form
{
"dataset": "EmnistLinesDataset",
"dataset_args": {
"max_overlap": 0.4,
"subsample_fraction": 0.2
},
"model": "LineModel",
"network": "line_cnn_all_conv",
"network_args": {
"window_width": 14,
"window_stride": 7
},
"train_args": {
"batch_size": 128,
"epochs": 10
}
}
save_weights (bool)
If True, will save the final model weights to a canonical location (see Model in models/base.py)
gpu_ind (int)
specifies which gpu to use (or -1 for first available)
use_wandb (bool)
sync training run to wandb
"""
print(f'Running experiment with config {experiment_config} on GPU {gpu_ind}')
datasets_module = importlib.import_module('text_recognizer.datasets')
dataset_class_ = getattr(datasets_module, experiment_config['dataset'])
dataset_args = experiment_config.get('dataset_args', {})
dataset = dataset_class_(**dataset_args)
dataset.load_or_generate_data()
print(dataset)
models_module = importlib.import_module('text_recognizer.models')
model_class_ = getattr(models_module, experiment_config['model'])
networks_module = importlib.import_module('text_recognizer.networks')
network_fn_ = getattr(networks_module, experiment_config['network'])
network_args = experiment_config.get('network_args', {})
model = model_class_(
dataset_cls=dataset_class_,
network_fn=network_fn_,
dataset_args=dataset_args,
network_args=network_args
)
print(model.network)
experiment_config['train_args'] = {**DEFAULT_TRAIN_ARGS, **experiment_config.get('train_args', {})}
experiment_config['minor_train_args'] = {k:v for k,v in experiment_config['train_args'].items() if k not in ('epochs', 'batch_size')}
experiment_config['experiment_group'] = experiment_config.get('experiment_group', None)
experiment_config['gpu_ind'] = gpu_ind
train_model(
model,
dataset,
epochs=experiment_config['train_args']['epochs'],
batch_size=experiment_config['train_args']['batch_size'],
gpu_ind=gpu_ind,
use_wandb=use_wandb,
**experiment_config['minor_train_args']
)
score = model.evaluate(dataset.x_test, dataset.y_test)
print(f'Test evaluation: {score}')
if save_weights:
model.save_weights()
def run_experiment(experiment_config: Dict,
save_weights: bool,
gpu_ind: int,
use_wandb: bool = True):
"""
experiment_config is of the form
{
"dataset": "EmnistLinesDataset",
"dataset_args": {
"max_overlap": 0.4
},
"model": "LineModel",
"network": "line_cnn_sliding_window",
"network_args": {
"window_width": 14,
"window_stride": 7
},
"train_args": {
"batch_size": 128,
"epochs": 10
}
}
save_weights: if True, will save the final model weights to a canonical location (see Model in models/base.py)
gpu_ind: integer specifying which gpu to use
"""
print(
f'Running experiment with config {experiment_config} on GPU {gpu_ind}')
# import Datasets module from repo. Make sure Python path set to main repo
datasets_module = importlib.import_module('text_recognizer.datasets')
dataset_class_ = getattr(datasets_module, experiment_config['dataset'])
dataset_args = experiment_config.get('dataset_args', {})
dataset = dataset_class_(**dataset_args)
dataset.load_or_generate_data()
print(dataset)
# Import Models module, instantiate model object with dataset and network objects
models_module = importlib.import_module('text_recognizer.models')
# CharacterModel, LineModel...etc.
model_class_ = getattr(models_module, experiment_config['model'])
networks_module = importlib.import_module('text_recognizer.networks')
# mlp, resnet...etc.
network_fn_ = getattr(networks_module, experiment_config['network'])
network_args = experiment_config.get('network_args', {})
# model object
model = model_class_(dataset_cls=dataset_class_,
network_fn=network_fn_,
dataset_args=dataset_args,
network_args=network_args)
print(model)
experiment_config['train_args'] = {
**DEFAULT_TRAIN_ARGS,
**experiment_config.get('train_args', {})
}
experiment_config['experiment_group'] = experiment_config.get(
'experiment_group', None)
experiment_config['gpu_ind'] = gpu_ind
# Run training script using loaded objects
train_model(model,
dataset,
epochs=experiment_config['train_args']['epochs'],
batch_size=experiment_config['train_args']['batch_size'],
gpu_ind=gpu_ind,
use_wandb=use_wandb)
# Compute performance using evaluation metric on the test set
score = model.evaluate(dataset.x_test, dataset.y_test)
# Print performance score
print(f'Test evaluation: {score}')
# Save model weights
if save_weights:
model.save_weights()
