def call_run(parser):
run_model(parser.prototxt,
parser.input_weights,
parser.image0,
parser.image1,
parser.out_flow,
verbose=False)
def main():
args = Parser().parse()
use_cuda = not args.cpu and torch.cuda.is_available()
device = torch.device(f"cuda:{args.gpu}" if use_cuda else "cpu")
torch.manual_seed(args.seed)
train_data, test_data, train_loader, test_loader = get_data(args)
# labels should be a whole number from [0, num_classes - 1]
num_labels = int(max(max(train_data.targets), max(test_data.targets))) + 1
output_size = num_labels
init_args = [args.normalize, not args.deterministic, device]
models = {
'lenet5': lenet5.LeNet5,
'simpleconv': simpleconv.SimpleConv,
'complexconv': complexconv.ComplexConv
}
model = models[args.model](*init_args).to(device)
print("Model Architecture: ", model)
print("Using device: ", device)
print("Train Data Shape: ", train_data.data.shape)
print("Normalize layer outputs?: ", args.normalize)
criterion = torch.nn.CrossEntropyLoss()
run_model(model, args, criterion, train_loader, test_loader, num_labels, device)
def run_square_hex_sb_testcase():
create_grids()
create_boundary_info()
create_ic()
run_model()
plot_testcase()
def run_stress_divergence_testcase():
get_testcase_data()
create_ic()
run_model()
stress_divergence_map()
stress_divergence_scaling()
def run_square_quadhex_testcase():
create_grids()
create_ics()
run_model()
set_difference_fields()
plot_testcase()
def run_strain_testcase():
get_testcase_data()
create_ic()
run_model()
average_variational_strains()
strain_map()
strain_scaling()
def evaluate(split, model_name, model_path, augment, use_gpu):
train_loader, valid_loader, test_loader = load_data(augment, use_gpu)
writer = SummaryWriter()
model = NetFactory.createNet(model_name)
state_dict = torch.load(model_path,
map_location=(None if use_gpu else 'cpu'))
model.load_state_dict(state_dict)
if use_gpu:
model = model.cuda()
if split == 'train':
loader = train_loader
elif split == 'valid':
loader = valid_loader
elif split == 'test':
loader = test_loader
else:
raise ValueError("split must be 'train', 'valid', or 'test'")
loss, auc, preds, labels = run_model(writer, 1, model, loader)
print(f'{split} loss: {loss:0.4f}')
print(f'{split} AUC: {auc:0.4f}')
return preds, labels
def airline_experiment(method, sparsity_factor, run_id,
image=None, n_threads=1, partition_size=3000,
optimize_stochastic=False):
name = 'airline'
data = data_source.airline_data()[run_id - 1]
kernel = get_kernels(data['train_inputs'].shape[1], 1, True)
cond_ll = likelihood.UnivariateGaussian(np.array(1.0))
transform = data_transformation.MeanStdTransformation(data['train_inputs'],
data['train_outputs'])
return run_model.run_model(data['train_inputs'],
data['train_outputs'],
data['test_inputs'],
data['test_outputs'],
cond_ll,
kernel,
method,
name,
data['id'],
sparsity_factor,
transform,
False,
False,
optimization_config={'mog': 5, 'hyp': 2, 'll': 1, 'inducing': 1},
max_iter=200,
partition_size=partition_size,
ftol=10,
n_threads=n_threads,
model_image_dir=image,
optimize_stochastic=optimize_stochastic)
def seismic_experiment(method, components, sparsity_factor, run_id,
image=None, n_threads=1, partition_size=3000,
optimize_stochastic=False):
name = 'seismic'
data = data_source.seismic_data()[0]
kernel = get_kernels(data['train_inputs'].shape[1], 8, True)
cond_ll = likelihood.SeismicLL(4)
transform = data_transformation.MeanStdTransformation(data['train_inputs'],
data['train_outputs'])
return run_model.run_model(data['train_inputs'],
data['train_outputs'],
data['test_inputs'],
data['test_outputs'],
cond_ll,
kernel,
method,
components,
name,
data['id'],
sparsity_factor,
transform,
False,
False,
optimization_config={'mog': 5, 'hyp': 2, 'inducing': 1},
max_iter=200,
partition_size=partition_size,
ftol=10,
n_threads=n_threads,
model_image_dir=image,
optimize_stochastic=optimize_stochastic)
def wisconsin_experiment(method, sparsity_factor, run_id):
"""
Run the wisconsin experiment.
Parameters
----------
method : str
The method under which to run the experiment (mix1, mix2, or full).
sparsity_factor : float
The sparsity of inducing points.
run_id : int
The id of the configuration.
"""
name = 'breast_cancer'
data = data_source.wisconsin_breast_cancer_data()[run_id - 1]
kernel = get_kernels(data['train_inputs'].shape[1], 1, False)
cond_ll = likelihood.LogisticLL()
transform = data_transformation.IdentityTransformation(data['train_inputs'],
data['train_outputs'])
return run_model.run_model(data['train_inputs'],
data['train_outputs'],
data['test_inputs'],
data['test_outputs'],
cond_ll,
kernel,
method,
name,
data['id'],
sparsity_factor,
transform,
True,
False,
optimization_config={'mog': 25, 'hyp': 25},
max_iter=200)
def boston_experiment(method, sparsity_factor, run_id):
"""
Run the boston housing experiment.
Parameters
----------
method : str
The method under which to run the experiment (mix1, mix2, or full).
sparsity_factor : float
The sparsity of inducing points.
run_id : int
The id of the configuration.
"""
name = 'boston'
data = data_source.boston_data()[run_id - 1]
kernel = get_kernels(data['train_inputs'].shape[1], 1, True)
cond_ll = likelihood.UnivariateGaussian(np.array(1.0))
transform = data_transformation.MeanTransformation(data['train_inputs'], data['train_outputs'])
return run_model.run_model(data['train_inputs'],
data['train_outputs'],
data['test_inputs'],
data['test_outputs'],
cond_ll,
kernel,
method,
name,
data['id'],
sparsity_factor,
transform,
True,
False,
optimization_config={'mog': 25, 'hyp': 25, 'll': 25, 'inducing': 8},
max_iter=200)
def airline_experiment(method, components, sparsity_factor, run_id,
image=None, n_threads=1, partition_size=3000,
optimize_stochastic=False):
name = 'airline'
data = data_source.airline_data()[run_id - 1]
kernel = get_kernels(data['train_inputs'].shape[1], 1, True)
cond_ll = likelihood.UnivariateGaussian(np.array(1.0))
transform = data_transformation.MeanStdTransformation(data['train_inputs'],
data['train_outputs'])
return run_model.run_model(data['train_inputs'],
data['train_outputs'],
data['test_inputs'],
data['test_outputs'],
cond_ll,
kernel,
method,
components,
name,
data['id'],
sparsity_factor,
transform,
False,
False,
optimization_config={'mog': 5, 'hyp': 2, 'll': 1, 'inducing': 1},
max_iter=200,
partition_size=partition_size,
ftol=10,
n_threads=n_threads,
model_image_dir=image,
optimize_stochastic=optimize_stochastic)
def mnist8m_experiment(method,
components,
sparsity_factor,
run_id,
image=None,
n_threads=1,
partition_size=3000,
optimize_stochastic=True,
num_samples=1000,
max_iter=8000):
"""
Run the mnist8m experiment.
Parameters
----------
method : str
The method under which to run the experiment (mix1, mix2, or full).
sparsity_factor : float
The sparsity of inducing points.
run_id : int
The id of the configuration.
"""
name = 'mnist8m'
data = data_source.mnist8m_data()[run_id - 1]
kernel = [
ExtRBF(data['train_inputs'].shape[1],
variance=11,
lengthscale=np.array((9., )),
ARD=False) for _ in range(10)
]
cond_ll = likelihood.SoftmaxLL(10)
transform = data_transformation.IdentityTransformation(
data['train_inputs'], data['train_outputs'])
return run_model.run_model(data['train_inputs'],
data['train_outputs'],
data['test_inputs'],
data['test_outputs'],
cond_ll,
kernel,
method,
components,
name,
data['id'],
sparsity_factor,
transform,
False,
False,
optimization_config={
'mog': 60,
'hyp': 15
},
num_samples=num_samples,
max_iter=max_iter,
n_threads=n_threads,
ftol=10,
model_image_dir=image,
partition_size=partition_size,
optimize_stochastic=optimize_stochastic)
def sarcos_all_joints_experiment(method,
components,
sparsity_factor,
run_id,
image=None,
n_threads=1,
partition_size=3000,
optimize_stochastic=False):
"""
Run the sarcos experiment on all joints.
Parameters
----------
method : str
The method under which to run the experiment (mix1, mix2, or full).
sparsity_factor : float
The sparsity of inducing points.
run_id : int
The id of the configuration.
"""
name = 'sarcos_all_joints'
data = data_source.sarcos_all_joints_data()[run_id - 1]
kernel = get_kernels(data['train_inputs'].shape[1], 8, False)
cond_ll = likelihood.CogLL(0.1, 7, 1)
scaler = data_transformation.preprocessing.StandardScaler().fit(
data['train_inputs'])
data['train_inputs'] = scaler.transform(data['train_inputs'])
data['test_inputs'] = scaler.transform(data['test_inputs'])
transform = data_transformation.MeanStdYTransformation(
data['train_inputs'], data['train_outputs'])
return run_model.run_model(data['train_inputs'],
data['train_outputs'],
data['test_inputs'],
data['test_outputs'],
cond_ll,
kernel,
method,
components,
name,
data['id'],
sparsity_factor,
transform,
False,
False,
optimization_config={
'mog': 25,
'hyp': 10,
'll': 10,
'inducing': 6
},
max_iter=200,
partition_size=partition_size,
ftol=10,
n_threads=n_threads,
model_image_dir=image,
optimize_stochastic=optimize_stochastic)
def experiment(params, param, param_values):
accuracies = []
new_params = {**default_params, **params}
for value in tqdm(param_values, desc='Parameter values'):
new_params[param] = value
print(value)
_, _, truth_values, predictions = run_model(new_params)
accuracy, _, _ = calc_derivations(truth_values, predictions)
accuracies.append(accuracy)
return accuracies
def get_data():
filepath = 'xgboost_input.csv'
if not os.path.exists(filepath):
create_input_data
input_data = pd.read_csv('xgboost_input.csv')
output_data, model = run_model.run_model()
results = run_model.results(input_data, output_data)
return results
def mnist_binary_inducing_experiment(method,
sparsity_factor,
run_id,
image=None,
n_threads=1,
partition_size=3000,
optimize_stochastic=False):
"""
Run the binary mnist experiment with inducing point learning.
Parameters
----------
method : str
The method under which to run the experiment (mix1, mix2, or full).
sparsity_factor : float
The sparsity of inducing points.
run_id : int
The id of the configuration.
"""
name = 'mnist_binary'
data = data_source.mnist_binary_data()[run_id - 1]
kernel = [
ExtRBF(data['train_inputs'].shape[1],
variance=11,
lengthscale=np.array((9., )),
ARD=False)
]
cond_ll = likelihood.LogisticLL()
transform = data_transformation.IdentityTransformation(
data['train_inputs'], data['train_outputs'])
return run_model.run_model(data['train_inputs'],
data['train_outputs'],
data['test_inputs'],
data['test_outputs'],
cond_ll,
kernel,
method,
name,
data['id'],
sparsity_factor,
transform,
False,
False,
optimization_config={
'mog': 60,
'hyp': 15,
'inducing': 6
},
max_iter=9,
n_threads=n_threads,
ftol=10,
model_image_dir=image,
partition_size=partition_size,
optimize_stochastic=optimize_stochastic)
def creep_experiment(method,
components,
sparsity_factor,
run_id,
optimize_stochastic=False):
"""
Run the creep experiment.
Parameters
----------
method : str
The method under which to run the experiment (mix1, mix2, or full).
sparsity_factor : float
The sparsity of inducing points.
run_id : int
The id of the configuration.
"""
name = 'creep'
data = data_source.creep_data()[run_id - 1]
kernel = get_kernels(data['train_inputs'].shape[1], 1, True)
cond_ll = likelihood.WarpLL(np.array([3.8715, 3.8898, 2.8759]),
np.array([1.5925, -1.3360, -2.0289]),
np.array([0.7940, -4.1855, -3.0289]),
np.log(0.01))
scaler = data_transformation.preprocessing.StandardScaler().fit(
data['train_inputs'])
data['train_inputs'] = scaler.transform(data['train_inputs'])
data['test_inputs'] = scaler.transform(data['test_inputs'])
transform = data_transformation.MinTransformation(data['train_inputs'],
data['train_outputs'])
return run_model.run_model(data['train_inputs'],
data['train_outputs'],
data['test_inputs'],
data['test_outputs'],
cond_ll,
kernel,
method,
components,
name,
data['id'],
sparsity_factor,
transform,
True,
False,
optimization_config={
'mog': 25,
'hyp': 25,
'll': 25
},
max_iter=200)
def test_model_on_image_pair(prototxt,
model_weights, img0_p, img1_p, prefix):
from run_model import run_model
"""
img0_p, img1_p : paths
prefix : path where the output files will be saved (flow, and expected image)
"""
flow_p = prefix + "-out.flo"
it1 = cv2.imread(img1_p)
it0 = cv2.imread(img0_p)
run_model(prototxt, model_weights, img0_p, img1_p, flow_p, verbose=False)
flow = optical_flow_lib.read_flo_file(flow_p)
expected_it0 = apply_flow_reverse(it1, flow)
cv2.imwrite(prefix + "-img0-expected.jpg", expected_it0)
cv2.imwrite(prefix + "-img0.jpg", it0)
cv2.imwrite(prefix + "-img1.jpg", it1)
optical_flow_lib.save_flow_image(flow, prefix + "-flo.png")
return flow_p
def sarcos_inducing_experiment(method,
sparsity_factor,
run_id,
image=None,
n_threads=1,
partition_size=3000,
optimize_stochastic=False):
"""
Run the sarcos experiment on two joints.
Parameters
----------
method : str
The method under which to run the experiment (mix1, mix2, or full).
sparsity_factor : float
The sparsity of inducing points.
run_id : int
The id of the configuration.
"""
name = 'sarcos'
data = data_source.sarcos_data()[run_id - 1]
kernel = get_kernels(data['train_inputs'].shape[1], 3, False)
cond_ll = likelihood.CogLL(0.1, 2, 1)
transform = data_transformation.MeanStdYTransformation(
data['train_inputs'], data['train_outputs'])
return run_model.run_model(data['train_inputs'],
data['train_outputs'],
data['test_inputs'],
data['test_outputs'],
cond_ll,
kernel,
method,
name,
data['id'],
sparsity_factor,
transform,
True,
False,
optimization_config={
'mog': 5,
'hyp': 2,
'll': 2,
'inducing': 1
},
max_iter=200,
partition_size=partition_size,
n_threads=n_threads,
model_image_dir=image,
optimize_stochastic=optimize_stochastic)
def usps_experiment(method,
components,
sparsity_factor,
run_id,
optimize_stochastic=False):
"""
Run the usps experiment.
Parameters
----------
method : str
The method under which to run the experiment (mix1, mix2, or full).
sparsity_factor : float
The sparsity of inducing points.
run_id : int
The id of the configuration.
"""
name = 'usps'
data = data_source.usps_data()[run_id - 1]
kernel = [
ExtRBF(data['train_inputs'].shape[1],
variance=2,
lengthscale=np.array((4., )),
ARD=False) for _ in range(3)
]
cond_ll = likelihood.SoftmaxLL(3)
transform = data_transformation.IdentityTransformation(
data['train_inputs'], data['train_outputs'])
return run_model.run_model(data['train_inputs'],
data['train_outputs'],
data['test_inputs'],
data['test_outputs'],
cond_ll,
kernel,
method,
components,
name,
data['id'],
sparsity_factor,
transform,
True,
False,
optimization_config={
'mog': 25,
'hyp': 25
},
max_iter=300)
def abalone_experiment(method,
components,
sparsity_factor,
run_id,
optimize_stochastic=False):
"""
Run the abalone experiment.
Parameters
----------
method : str
The method under which to run the experiment (mix1, mix2, or full).
sparsity_factor : float
The sparsity of inducing points.
run_id : int
The id of the configuration.
"""
name = 'abalone'
data = data_source.abalone_data()[run_id - 1]
kernel = get_kernels(data['train_inputs'].shape[1], 1, False)
cond_ll = likelihood.WarpLL(np.array([-2.0485, 1.7991, 1.5814]),
np.array([2.7421, 0.9426, 1.7804]),
np.array([0.1856, 0.7024, -0.7421]),
np.log(0.1))
transform = data_transformation.MinTransformation(data['train_inputs'],
data['train_outputs'])
return run_model.run_model(data['train_inputs'],
data['train_outputs'],
data['test_inputs'],
data['test_outputs'],
cond_ll,
kernel,
method,
components,
name,
data['id'],
sparsity_factor,
transform,
True,
False,
optimization_config={
'mog': 25,
'hyp': 25,
'll': 25
},
max_iter=200)
def boston_experiment(method,
components,
sparsity_factor,
run_id,
optimize_stochastic=False):
"""
Run the boston housing experiment.
Parameters
----------
method : str
The method under which to run the experiment (mix1, mix2, or full).
sparsity_factor : float
The sparsity of inducing points.
run_id : int
The id of the configuration.
"""
name = 'boston'
data = data_source.boston_data()[run_id - 1]
kernel = get_kernels(data['train_inputs'].shape[1], 1, True)
cond_ll = likelihood.UnivariateGaussian(np.array(1.0))
transform = data_transformation.MeanTransformation(data['train_inputs'],
data['train_outputs'])
return run_model.run_model(data['train_inputs'],
data['train_outputs'],
data['test_inputs'],
data['test_outputs'],
cond_ll,
kernel,
method,
components,
name,
data['id'],
sparsity_factor,
transform,
True,
False,
optimization_config={
'mog': 25,
'hyp': 25,
'll': 25,
'inducing': 8
},
max_iter=200,
optimize_stochastic=optimize_stochastic)
def sarcos_all_joints_experiment(method, sparsity_factor, run_id,
image=None, n_threads=1, partition_size=3000,
optimize_stochastic=False):
"""
Run the sarcos experiment on all joints.
Parameters
----------
method : str
The method under which to run the experiment (mix1, mix2, or full).
sparsity_factor : float
The sparsity of inducing points.
run_id : int
The id of the configuration.
"""
name = 'sarcos_all_joints'
data = data_source.sarcos_all_joints_data()[run_id - 1]
kernel = get_kernels(data['train_inputs'].shape[1], 8, False)
cond_ll = likelihood.CogLL(0.1, 7, 1)
scaler = data_transformation.preprocessing.StandardScaler().fit(data['train_inputs'])
data['train_inputs'] = scaler.transform(data['train_inputs'])
data['test_inputs'] = scaler.transform(data['test_inputs'])
transform = data_transformation.MeanStdYTransformation(data['train_inputs'],
data['train_outputs'])
return run_model.run_model(data['train_inputs'],
data['train_outputs'],
data['test_inputs'],
data['test_outputs'],
cond_ll,
kernel,
method,
name,
data['id'],
sparsity_factor,
transform,
False,
False,
optimization_config={'mog': 25, 'hyp': 10, 'll': 10, 'inducing': 6},
max_iter=200,
partition_size=partition_size,
ftol=10,
n_threads=n_threads,
model_image_dir=image,
optimize_stochastic=optimize_stochastic)
def wisconsin_experiment(method,
components,
sparsity_factor,
run_id,
optimize_stochastic=False):
"""
Run the wisconsin experiment.
Parameters
----------
method : str
The method under which to run the experiment (mix1, mix2, or full).
sparsity_factor : float
The sparsity of inducing points.
run_id : int
The id of the configuration.
"""
name = 'breast_cancer'
data = data_source.wisconsin_breast_cancer_data()[run_id - 1]
kernel = get_kernels(data['train_inputs'].shape[1], 1, False)
cond_ll = likelihood.LogisticLL()
transform = data_transformation.IdentityTransformation(
data['train_inputs'], data['train_outputs'])
return run_model.run_model(data['train_inputs'],
data['train_outputs'],
data['test_inputs'],
data['test_outputs'],
cond_ll,
kernel,
method,
components,
name,
data['id'],
sparsity_factor,
transform,
True,
False,
optimization_config={
'mog': 25,
'hyp': 25
},
max_iter=200)
def mnist_experiment(method, sparsity_factor, run_id,
image=None, n_threads=1, partition_size=3000,
optimize_stochastic=False):
"""
Run the mnist experiment.
Parameters
----------
method : str
The method under which to run the experiment (mix1, mix2, or full).
sparsity_factor : float
The sparsity of inducing points.
run_id : int
The id of the configuration.
"""
name = 'mnist'
data = data_source.mnist_data()[run_id - 1]
kernel = [ExtRBF(data['train_inputs'].shape[1], variance=11, lengthscale=np.array((9.,)), ARD=False)
for _ in range(10)]
cond_ll = likelihood.SoftmaxLL(10)
transform = data_transformation.IdentityTransformation(data['train_inputs'],
data['train_outputs'])
return run_model.run_model(data['train_inputs'],
data['train_outputs'],
data['test_inputs'],
data['test_outputs'],
cond_ll,
kernel,
method,
name,
data['id'],
sparsity_factor,
transform,
False,
False,
optimization_config={'mog': 60, 'hyp': 15},
max_iter=300,
n_threads=n_threads,
ftol=10,
model_image_dir=image,
partition_size=partition_size,
optimize_stochastic=optimize_stochastic)
def mining_experiment(method,
components,
sparsity_factor,
run_id,
optimize_stochastic=False):
"""
Run the mining experiment.
Parameters
----------
method : str
The method under which to run the experiment (mix1, mix2, or full).
sparsity_factor : float
The sparsity of inducing points.
run_id : int
The id of the configuration.
"""
name = 'mining'
data = data_source.mining_data()[run_id - 1]
kernel = get_kernels(data['train_inputs'].shape[1], 1, False)
cond_ll = likelihood.LogGaussianCox(np.log(191. / 811))
transform = data_transformation.IdentityTransformation(
data['train_inputs'], data['train_outputs'])
kernel[0].variance = 1.0
kernel[0].lengthscale = 13516.0
return run_model.run_model(data['train_inputs'],
data['train_outputs'],
data['test_inputs'],
data['test_outputs'],
cond_ll,
kernel,
method,
components,
name,
data['id'],
sparsity_factor,
transform,
True,
True,
optimization_config={'mog': 15000},
max_iter=1)
def creep_experiment(method, sparsity_factor, run_id):
"""
Run the creep experiment.
Parameters
----------
method : str
The method under which to run the experiment (mix1, mix2, or full).
sparsity_factor : float
The sparsity of inducing points.
run_id : int
The id of the configuration.
"""
name = 'creep'
data = data_source.creep_data()[run_id - 1]
kernel = get_kernels(data['train_inputs'].shape[1], 1, True)
cond_ll = likelihood.WarpLL(np.array([3.8715, 3.8898, 2.8759]),
np.array([1.5925, -1.3360, -2.0289]),
np.array([0.7940, -4.1855, -3.0289]),
np.log(0.01))
scaler = data_transformation.preprocessing.StandardScaler().fit(data['train_inputs'])
data['train_inputs'] = scaler.transform(data['train_inputs'])
data['test_inputs'] = scaler.transform(data['test_inputs'])
transform = data_transformation.MinTransformation(data['train_inputs'], data['train_outputs'])
return run_model.run_model(data['train_inputs'],
data['train_outputs'],
data['test_inputs'],
data['test_outputs'],
cond_ll,
kernel,
method,
name,
data['id'],
sparsity_factor,
transform,
True,
False,
optimization_config={'mog': 25, 'hyp': 25, 'll': 25},
max_iter=200)
def abalone_experiment(method, sparsity_factor, run_id):
"""
Run the abalone experiment.
Parameters
----------
method : str
The method under which to run the experiment (mix1, mix2, or full).
sparsity_factor : float
The sparsity of inducing points.
run_id : int
The id of the configuration.
"""
name = 'abalone'
data = data_source.abalone_data()[run_id - 1]
kernel = get_kernels(data['train_inputs'].shape[1], 1, False)
cond_ll = likelihood.WarpLL(np.array([-2.0485, 1.7991, 1.5814]),
np.array([2.7421, 0.9426, 1.7804]),
np.array([0.1856, 0.7024, -0.7421]),
np.log(0.1))
transform = data_transformation.MinTransformation(data['train_inputs'], data['train_outputs'])
return run_model.run_model(data['train_inputs'],
data['train_outputs'],
data['test_inputs'],
data['test_outputs'],
cond_ll,
kernel,
method,
name,
data['id'],
sparsity_factor,
transform,
True,
False,
optimization_config={'mog': 25, 'hyp': 25, 'll': 25},
max_iter=200)
def mining_experiment(method, sparsity_factor, run_id):
"""
Run the mining experiment.
Parameters
----------
method : str
The method under which to run the experiment (mix1, mix2, or full).
sparsity_factor : float
The sparsity of inducing points.
run_id : int
The id of the configuration.
"""
name = 'mining'
data = data_source.mining_data()[run_id - 1]
kernel = get_kernels(data['train_inputs'].shape[1], 1, False)
cond_ll = likelihood.LogGaussianCox(np.log(191. / 811))
transform = data_transformation.IdentityTransformation(data['train_inputs'],
data['train_outputs'])
kernel[0].variance = 1.0
kernel[0].lengthscale = 13516.0
return run_model.run_model(data['train_inputs'],
data['train_outputs'],
data['test_inputs'],
data['test_outputs'],
cond_ll,
kernel,
method,
name,
data['id'],
sparsity_factor,
transform,
True,
True,
optimization_config={'mog': 15000},
max_iter=1)
def usps_experiment(method, sparsity_factor, run_id):
"""
Run the usps experiment.
Parameters
----------
method : str
The method under which to run the experiment (mix1, mix2, or full).
sparsity_factor : float
The sparsity of inducing points.
run_id : int
The id of the configuration.
"""
name = 'usps'
data = data_source.usps_data()[run_id - 1]
kernel = [ExtRBF(data['train_inputs'].shape[1], variance=2, lengthscale=np.array((4.,)), ARD=False)
for _ in range(3)]
cond_ll = likelihood.SoftmaxLL(3)
transform = data_transformation.IdentityTransformation(data['train_inputs'],
data['train_outputs'])
return run_model.run_model(data['train_inputs'],
data['train_outputs'],
data['test_inputs'],
data['test_outputs'],
cond_ll,
kernel,
method,
name,
data['id'],
sparsity_factor,
transform,
True,
False,
optimization_config={'mog': 25, 'hyp': 25},
max_iter=300)
from create_grid import create_grid from run_model import run_model from plot_testcase import plot_testcase from plot_thicknesses import plot_thicknesses create_grid() run_model() plot_testcase() plot_thicknesses()
from run_model import run_model
from domain_model import set_skip_log as skip_current_action
class SkipLogAdvice(object):
def should_skip(self, attr):
'''
Skip all scheduling actions.
:return: Bool indicating whether to activate.
'''
return 'schedule' in attr.func_name
def prelude(self, attr, _):
if self.should_skip(attr):
skip_current_action(True)
run_model(SkipLogAdvice)
# Check the integrity of the data
is_data_current(regions)
do_daily_cases_increase(regions)
# # Quick test to see delay curve for a single region
# # Let's look at the adjusted data for a single region - South East
# # (contains Oxford)
# region = 'South East'
# confirmed = regions.xs(region)['Cumulative lab-confirmed cases'].diff().dropna()
# # confirmed.tail()
#
#
# # Compute the adjustment using the probability of delay - p_delay
# onset = confirmed_to_onset(confirmed, p_delay)
# adjusted, cumulative_p_delay = adjust_onset_for_right_censorship(onset, p_delay)
#
#
# # Plot the data for our our single region
# # plot_adjusted_data(region,confirmed,onset,adjusted)
run_model(p_delay, regions)
# Plot charts
plot_rt()
# Copy update date to web page
### Compare to other analyses
# compare.compare_epiforecasts()
def train(train_loader,
valid_loader,
epochs=20,
learning_rate=2e-5,
regularization=0.01,
eps=1e-8,
model=None,
device="cuda",
loss_weights=None,
loss_func=None,
save_path="save models",
model_name=None):
format_time(time.time() - time.time())
model.to(device)
optimizer = AdamW(model.parameters(), lr=learning_rate, eps=eps)
total_steps = len(train_loader) * epochs
scheduler = get_linear_schedule_with_warmup(optimizer,
num_warmup_steps=0,
num_training_steps=total_steps)
seed_val = 42
random.seed(seed_val)
np.random.seed(seed_val)
torch.manual_seed(seed_val)
torch.cuda.manual_seed_all(seed_val)
# We'll store a number of quantities such as training and validation loss,
# validation accuracy, and timings.
training_stats = []
# Measure the total training time for the whole run.
total_t0 = time.time()
# if loss func not specified use model 's own loss
if loss_func == "weighted_CrossEntropy":
print("using weighted_CrossEntropy loss")
# loss_func=nn.CrossEntropyLoss(weight=loss_weights,size_average=False)
loss_func = nn.CrossEntropyLoss()
best_valid_f1 = 0
best_valid_preds = []
# For each epoch...
for epoch_i in range(epochs):
# ========================================
# Training
# ========================================
# Perform one full pass over the training set.
print("")
print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))
# print('Training...')
model.to(device)
# Measure how long the training epoch takes.
t0 = time.time()
training_loss, training_acc, training_f1, training_recall, training_preds, training_labels = run_model(
model,
train_loader,
True,
optimizer,
scheduler,
device=device,
loss_func=loss_func)
print(" Average training loss: {:.6f}".format(training_loss))
print(" Average training accuracy: {0:.4f}".format(training_acc))
print(" Average training f1: {0:.4f}".format(training_f1))
print(" Average training recall: {0:.4f}".format(training_recall))
print("-" * 50)
valid_loss, valid_acc, valid_f1, valid_recall, valid_preds, valid_labels = run_model(
model, valid_loader, device=device, loss_func=loss_func)
print(" Average validation loss: {0:.4f}".format(valid_loss))
print(" Average validation accuracy: {0:.4f}".format(valid_acc))
print(" Average validation f1: {0:.4f}".format(valid_f1))
print(" Average validation recall: {0:.4f}".format(valid_recall))
print("-" * 50)
# Measure how long this epoch took.
training_time = format_time(time.time() - t0)
def main(args):
""" Main function of the program
Inputs:
args: the input arguments of the program in the form of a dictionary {"image" : <argument>}.
if args exist, <argument> is the input image, else <argument> is None.
Output: None
"""
# Define dataset directory
data_dir = "data"
# Define dataset files
trainset_name = "train.p"
validset_name = "valid.p"
testset_name = "test.p"
# Finding dataset properties
class_names = class_names_fun(data_dir)
# Visualizing the dataset
dataset_properties(trainset_name, validset_name, testset_name, class_names,
data_dir)
# Define the device parameters
torch.manual_seed(1)
use_cuda = torch.cuda.is_available()
device = torch.device("cuda" if use_cuda else "cpu")
# Define the model
model = BaselineNet().to(device)
# Define the training properties
epoch_num = 100
criterion = nn.CrossEntropyLoss()
learning_rate = 1e-3
batch_size = 64
stop_threshold = 1e-4
# Computing data transformation to normalize data
# from https://pytorch.org/docs/stable/torchvision/transforms.html
mean = (0.485, 0.456, 0.406)
std = (0.229, 0.224, 0.225) # -"-
transform = transforms.Compose([
transforms.ToTensor(),
transforms.Resize((32, 32)),
transforms.Normalize(mean=mean, std=std)
])
# Reading the datasets
train_dataset = ReadDataset(trainset_name, transform=transform)
valid_dataset = ReadDataset(validset_name, transform=transform)
test_dataset = ReadDataset(testset_name, transform=transform)
# If no input model - training a new model
if not args["model"]:
# Defining the model
model_path = os.path.abspath("model")
# Train the network
model, train_loss_list, valid_loss_list, valid_accuracy_list = run_model(
model,
running_mode='train',
train_set=train_dataset,
valid_set=valid_dataset,
test_set=test_dataset,
batch_size=batch_size,
epoch_num=epoch_num,
learning_rate=learning_rate,
stop_thr=stop_threshold,
criterion=criterion,
device=device)
# Plot the results of training the network
plot_training_results(train_loss_list, valid_loss_list,
valid_accuracy_list, epoch_num)
# Save the trained model
torch.save(model.state_dict(), model_path)
# If input model - load the existing model
else:
# Defining the model
model_path = os.path.abspath(args["model"])
# Load the trained model
model.load_state_dict(torch.load(model_path, map_location=device))
# Test the network
test_loss, test_accuracy = run_model(model,
running_mode='test',
train_set=train_dataset,
valid_set=valid_dataset,
test_set=test_dataset,
batch_size=batch_size,
epoch_num=epoch_num,
learning_rate=learning_rate,
stop_thr=stop_threshold,
criterion=criterion,
device=device)
print(f"Test loss: {test_loss:.3f}")
print(f"Test accuracy: {test_accuracy:.2f}%")
# Check if image argument exists
if args["image"]:
# Load the image argument
test_image = load_arguments(args)
test_image_resized = cv2.resize(test_image, (32, 32))
test_image_tensor = transforms.ToTensor()(np.array(test_image_resized))
# Transform tested image
test_image_transform4d = test_image_tensor.unsqueeze(0)
# Predict the class of the tested image
prediction = int(predict(model, test_image_transform4d)[0])
print(
f"Test prediction: {prediction} -> Class: {class_names[prediction]}"
)
# Plot the image with the predicted class
plt.figure()
plt.axis('off')
plt.title(class_names[prediction], fontsize=10)
plt.imshow(test_image)
plt.suptitle('Image Classification', fontsize=18)
plt.savefig('images/Image_Classification')
plt.show()
def train_and_eval(model, train_loader, valid_loader, learning_rate, epochs,
model_outdir, #pos_wt,
metrics_every_iter, task, tensorboard = False,
restore_chkpt = None, run_suffix = None):
"""
Contains the powerhouse of the network, ie. the training and validation iterations called through run_model().
All parameters from the command line/json are parsed and then passed into run_model().
Performs checkpointing each epoch, saved as 'last.pth.tar' and the best model thus far (based on validation AUC), saved as 'best.pth.tar'
:param model: (nn.Module) -
:param train_loader: (torch DataLoader)
:param valid_loader: (torch DataLoader)
:param learning_rate: (float) - the learning rate, defaults to 1e-05
:param epochs: (int) - the number of epochs
:param model_outdir: (str) - the output directory for checkpointing, checkpoints will be saved as output_dir/task/view/*.tar
:param restore_chkpt: (str) - the directory to reload the checkpoint, if specified
:param run_suffix: (str) - suffix to be appended to the event file. removed for now.
:return:
"""
log_fn = helpers.create_tb_log_dir(model_outdir)
log_fn = log_fn.strip("/") # remove leading forward slash which messes up tf log
if tensorboard:
import tensorflow as tf
writer = tf.summary.create_file_writer(log_fn) # tf 2.0+
writer = tf.compat.v1.summary.FileWriter(log_fn) # tf v1.15
current_best_val_loss = float('Inf')
optimizer = torch.optim.Adam(model.parameters(), learning_rate, weight_decay=0.01)
# taken directly from MRNet code
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer,
patience = 5, # how many epochs to wait for before acting
factor = 0.3, # factor to reduce LR by, LR = factor * LR
threshold = 1e-4) # threshold to measure new optimum
#
losses = {'regression': torch.nn.MSELoss(),
'classification': torch.nn.BCEWithLogitsLoss(),
'multitask': torch.nn.MSELoss()}
#
criterion = losses[task]
print(criterion)
metric = {'regression':'MSE',
'classification':'AUC',
'multitask': 'MSE'} # Steve: Seems like this is mostly for logging?
# # TODO: reloading checkpoint
# if restore_chkpt:
# logging.info("Restoring Checkpoint from {}".format(restore_chkpt))
# helpers.load_checkpoint(checkpoint = restore_chkpt,
# model = model,
# optimizer = optimizer,
# # scheduler = scheduler,
# epochs = epochs)
# # print(loaded_epoch)
# # so epochs - loaded_epoch is where we would need to start, right?
# logging.info("Starting again at Epoch {}....".format(epochs))
# logging.info("Finished Restoring Checkpoint...")
for epoch in range(epochs):
logging.info('[Epoch {}]'.format(epoch + 1))
# main training loop
epoch_loss, epoch_metric, epoch_preds, epoch_labels, train_df = run_model(
model = model,
loader = train_loader,
optimizer = optimizer,
criterion = criterion,
metrics_every_iter = metrics_every_iter,
task = task,
tensorboard = tensorboard,
train = True)
logging.info('[Epoch {}]\t\tTraining {}: {:.3f}\t Training Average Loss: {:.5f}'\
.format(epoch + 1, metric[task], epoch_metric, epoch_loss))
# main validation loop
epoch_val_loss, epoch_val_metric, epoch_val_preds, epoch_val_labels, val_df = run_model(model = model,
loader = valid_loader,
optimizer = optimizer,
criterion = criterion,
task = task,
tensorboard = tensorboard,
metrics_every_iter = False, # default, just show the epoch validation metrics..
train = False)
logging.info('[Epoch {}]\t\tValidation {}: {:.3f}\t Validation Average Loss: {:.5f}'.format(epoch + 1, metric[task], epoch_val_metric, epoch_val_loss))
scheduler.step(epoch_val_loss) # check per epoch, how does the threshold work?!?!?
logging.info('[Epoch {}]\t\tOptimizer Learning Rate: {}'.format(epoch + 1, {optimizer.param_groups[0]['lr']}))
# with writer:#.as_default():
# temp = torch.tensor([epoch + 1]) # needs to be a tesor in tf v1.5?
# writer.add_summary(tf.compat.v1.summary.scalar('Loss/train', epoch_loss), temp).eval()
# writer.add_summary(tf.compat.v1.summary.scalar('Loss/val', epoch_val_loss), temp).eval()
# writer.add_summary(tf.compat.v1.summary.scalar('{}/train'.format(metric[task]), epoch_metric), temp).eval()
# writer.add_summary(tf.compat.v1.summary.scalar('{}/val'.format(metric[task]), epoch_val_metric), temp).eval()
# writer_flush = writer.flush()
# with writer.as_default():
# tf.summary.scalar('Loss/train', epoch_loss, epoch + 1)
# tf.summary.scalar('Loss/val', epoch_val_loss, epoch + 1)
# tf.summary.scalar('{}/train'.format(metric[task]), epoch_metric, epoch + 1)
# tf.summary.scalar('{}/val'.format(metric[task]), epoch_val_metric, epoch + 1)
print('Loss/train: {} for epoch: {}'.format(str(epoch_loss), str(epoch + 1)))
print('Loss/val: {} for epoch: {}'.format(str(epoch_val_loss), str(epoch + 1)))
print('{}/train: {} for epoch: {}'.format(metric[task], str(epoch_metric), str(epoch + 1)))
print('{}/val: {} for epoch: {}'.format(metric[task], str(epoch_val_metric), str(epoch + 1)))
# check whether the most recent epoch loss is better than previous best
is_best_val_loss = epoch_val_loss < current_best_val_loss
# save state in a dictionary
state = {'epoch': epoch + 1,
'state_dict': model.state_dict(),
'validation_metric': epoch_val_metric,
'metric': metric[task],
'best_validation_loss': epoch_val_loss,
# 'metrics': metrics # read more into this
'scheduler_dict': scheduler.state_dict(),
'optim_dict': optimizer.state_dict()}
# save as last epoch
helpers.save_checkpoint(state,
is_best = is_best_val_loss,
checkpoint_dir = model_outdir)
if is_best_val_loss:
current_best_val_loss = epoch_val_loss
logging.info('[Epoch {}]\t\t******New Best Validation Loss: {:.3f}******'.format(epoch + 1, epoch_val_loss))
helpers.save_checkpoint(state,
is_best = is_best_val_loss,
checkpoint_dir = model_outdir)
#if task == 'multitask': # Steven: Seems like this should work the same if doing regression or classification. I'll try doing the same for regression by commenting out this if statement.
# train_df.to_csv(os.path.join(model_outdir, 'best_epoch_training_results.csv'))
# val_df.to_csv(os.path.join(model_outdir, 'best_epoch_validation_results.csv'))
train_df.to_csv(os.path.join(model_outdir, 'best_epoch_training_results.csv'))
val_df.to_csv(os.path.join(model_outdir, 'best_epoch_validation_results.csv'))
def train_and_eval(model,
train_loader,
valid_loader,
learning_rate,
epochs,
model_outdir,
wts,
task,
metrics_every_iter,
restore_chkpt=None,
run_suffix=None):
"""
Contains the powerhouse of the network, ie. the training and validation iterations called through run_model().
All parameters from the command line/json are parsed and then passed into run_model().
Performs checkpointing each epoch, saved as 'last.pth.tar' and the best model thus far (based on validation AUC), saved as 'best.pth.tar'
:param model: (nn.Module)
:param train_loader: (torch DataLoader)
:param valid_loader: (torch DataLoader)
:param learning_rate: (float) - the learning rate, defaults to 1e-05
:param epochs: (int) - the number of epochs
:param wts: (tensor) - class weights
:param model_outdir: (str) - the output directory for checkpointing, checkpoints will be saved as output_dir/task/view/*.tar
:param restore_chkpt: (str) - the directory to reload the checkpoint, if specified
:param run_suffix: (str) - suffix to be appended to the event file
:return:
"""
# output/task/my_run
# goes back 3 levels up, putting the name in the same level as the output. two levels up would put them into output/
recover_root_dir = os.path.dirname(
os.path.dirname(os.path.dirname(
model_outdir))) # removes the task and view from the directory
log_dir = os.path.join(recover_root_dir, "logs")
run_name = re.split(r'/|\\', model_outdir)[-1]
# task = re.split(r'/|\\', model_outdir)[-2]
# have log folder naming structure same as models
log_fn = os.path.join(log_dir, task, run_name)
dtnow = datetime.now()
# dtnow.strftime("%Y%m%d_%H%M%S")
log_fn = os.path.join(log_fn, dtnow.strftime("%Y_%m_%d-%H_%M_%S"))
# make directory if it doesn't exist.
if not os.path.exists(log_fn):
os.makedirs(log_fn)
print('{} does not exist, creating..!'.format(log_fn))
else:
print('{} already exists!'.format(log_fn))
# each tensorboard event file should ideally be saved to a unique folder, else the resulting graph will look like
# it's time traveling because of overlapping logs
# if run_suffix:
# writer = tf.summary.create_file_writer(log_fn, filename_suffix=run_suffix)
# else:
# writer = tf.summary.create_file_writer(log_fn)
# use cpu or cuda depending on availability
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
current_best_val_loss = float('Inf')
# this needs to be outside of the loop else it'll keep resetting, right? same with the model
optimizer = torch.optim.Adam(model.parameters(),
learning_rate,
weight_decay=0.01)
# taken directly from MRNet code
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
optimizer,
patience=5, # how many epochs to wait for before acting
factor=0.3, # factor to reduce LR by, LR = factor * LR
threshold=1e-4) # threshold to measure new optimum
# weight loss by training class positive weights, if use_wts is False then no weights are applied
# criterion_d = {'bladder': torch.nn.BCEWithLogitsLoss(), 'view': torch.nn.CrossEntropyLoss(), 'granular':torch.nn.CrossEntropyLoss()}
if wts is None:
criterion = torch.nn.CrossEntropyLoss()
else:
wts = wts.to(device)
criterion = torch.nn.CrossEntropyLoss(weight=wts)
# # TODO: reloading checkpoint
# if restore_chkpt:
# logging.info("Restoring Checkpoint from {}".format(restore_chkpt))
# helpers.load_checkpoint(checkpoint = restore_chkpt,
# model = model,
# optimizer = optimizer,
# # scheduler = scheduler,
# epochs = epochs)
# # so epochs - loaded_epoch is where we would need to start, right?
# logging.info("Starting again at Epoch {}....".format(epochs))
# logging.info("Finished Restoring Checkpoint...")
for epoch in range(epochs):
logging.info('[Epoch {}]'.format(epoch + 1))
# main training loop
epoch_loss, epoch_preds, epoch_labels = run_model(
model=model,
loader=train_loader,
optimizer=optimizer,
criterion=criterion,
metrics_every_iter=metrics_every_iter,
train=True)
logging.info('[Epoch {}]\t\t Training Average Loss: {:.5f}'.format(
epoch + 1, epoch_loss))
# logging.info('[Epoch {}]\t\tTraining Balanced Accuracy: {:.3f}\t Training Average Loss: {:.5f}'.format(epoch + 1, epoch_auc, epoch_loss))
# main validation loop
epoch_val_loss, epoch_val_preds, epoch_val_labels = run_model(
model=model,
loader=valid_loader,
optimizer=optimizer,
criterion=criterion,
metrics_every_iter=
False, # default, just show the epoch validation metrics..
train=False)
logging.info('[Epoch {}]\t\t Validation Average Loss: {:.5f}'.format(
epoch + 1, epoch_val_loss))
# logging.info('[Epoch {}]\t\tValidation Balanced Accuracy: {:.3f}\t Validation Average Loss: {:.5f}'.format(epoch + 1, epoch_val_acc, epoch_val_loss))
scheduler.step(epoch_val_loss
) # check per epoch, how does the threshold work?!?!?
logging.info('[Epoch {}]\t\tOptimizer Learning Rate: {}'.format(
epoch + 1, {optimizer.param_groups[0]['lr']}))
# with writer.as_default():
# tf.summary.scalar('Loss/train', epoch_loss, epoch + 1)
# tf.summary.scalar('Loss/val', epoch_val_loss, epoch + 1)
# tf.summary.scalar('BACC/train', epoch_acc, epoch + 1)
# tf.summary.scalar('BACC/val', epoch_val_acc, epoch + 1)
# check whether the most recent epoch loss is better than previous best
# is_best_val_auc = epoch_val_auc >= current_best_val_auc
is_best_val_loss = epoch_val_loss < current_best_val_loss
# save state in a dictionary
state = {
'epoch': epoch + 1,
'state_dict': model.state_dict(),
# 'validation_acc': epoch_val_acc,
'best_validation_loss': epoch_val_loss,
# 'metrics': metrics # read more into this
'scheduler_dict': scheduler.state_dict(),
'optim_dict': optimizer.state_dict()
}
# save as last epoch
helpers.save_checkpoint(state,
is_best=is_best_val_loss,
checkpoint_dir=model_outdir)
# epoch = epoch + 1)
if is_best_val_loss:
# set new best validation loss
# current_best_val_auc = epoch_val_auc
current_best_val_loss = epoch_val_loss
# logging.info('[Epoch {}]\t\t******New Best Validation:\t AUC: {:.3f}******'.format(epoch + 1, epoch_val_auc))
logging.info(
'[Epoch {}]\t\t******New Best Validation Loss: {:.3f}******'.
format(epoch + 1, epoch_val_loss))
helpers.save_checkpoint(state,
is_best=is_best_val_loss,
checkpoint_dir=model_outdir)
def seismic_experiment(method,
components,
sparsity_factor,
run_id,
image=None,
n_threads=1,
partition_size=3000,
optimize_stochastic=False):
name = 'seismic'
data = data_source.seismic_data()[0]
# prior_var = np.array([900, 5625, 57600, 108900, 38025, 52900, 75625, 133225])
# prior_mu = [200, 500, 1600, 2200, 1950, 2300, 2750, 3650]
# sigma2y = [0.0006, 0.0025, 0.0056, 0.0100]
scale_factor = 10. # for numerical reasons (final predicitons to be post-processed)
mean_depth = np.array([200.0, 500.0, 1600.0, 2200.0], dtype=np.double)
mean_vel = np.array([1950.0, 2300.0, 2750.0, 3650.0], dtype=np.double)
std_depth = mean_depth * 0.15
std_vel = mean_vel * 0.10
prior_mu = np.hstack((mean_depth, mean_vel)) / scale_factor
prior_var = np.square(np.hstack(
(std_depth, std_vel))) / (scale_factor * scale_factor)
sigma2y = np.square([0.025, 0.05, 0.075, 0.1])
input_dim = data['train_inputs'].shape[1]
kernel = [
ExtRBF(input_dim,
variance=prior_var[i],
lengthscale=np.array((1, )),
ARD=True) for i in range(len(prior_var))
]
cond_ll = likelihood.SeismicLL(4, sigma2y)
transform = data_transformation.IdentityTransformation(
data['train_inputs'], data['train_outputs'])
#transform = data_transformation.MeanStdXTransformation(data['train_inputs'], data['train_outputs'])
return run_model.run_model(
data['train_inputs'],
data['train_outputs'],
data['test_inputs'],
data['test_outputs'],
cond_ll,
kernel,
method,
components,
name,
data['id'],
sparsity_factor,
transform,
False,
True,
optimization_config={'mog': 100},
# max_iter=10,
partition_size=partition_size,
# ftol=1,
n_threads=n_threads,
model_image_dir=image,
GP_mean=prior_mu,
init_var=0.001 * prior_var,
num_samples=100000,
)
