# us-east-1 AMIs
# numpy00: ami-f9d6dc83
# numpy01: ami-5b524f21
from collections import OrderedDict
import argparse
import os
import sys
import time
import boto3
module_path = os.path.dirname(os.path.abspath(__file__))
sys.path.append(module_path + '/..')
import util
util.install_pdb_handler()
parser = argparse.ArgumentParser(description='launch')
parser.add_argument('--ami',
type=str,
default='ami-5b524f21',
help="name of AMI to use ")
parser.add_argument('--group',
type=str,
default='dawn_runs',
help="name of the current run")
parser.add_argument('--name',
type=str,
default='baseline5-tong',
help="name of the current run")
parser.add_argument('--instance-type',
from collections import OrderedDict
import argparse
import os
import sys
import time
import collections
import boto3
import datetime
import threading
module_path = os.path.dirname(os.path.abspath(__file__))
sys.path.append(module_path + '/..')
import util as u
import aws_backend
import launch_utils as launch_utils_lib
u.install_pdb_handler()
parser = argparse.ArgumentParser(description='launch')
parser.add_argument('--ami-name',
type=str,
default='-1',
help="name of AMI to use")
parser.add_argument('--spot',
action='store_true',
help='launch using spot requests')
parser.add_argument('--name',
type=str,
default='imagenet',
help=("name of the current run, this determines placement "
"group name, instance names and EFS logging "
"directory."))
#!/usr/bin/env python
# script to launch cifar-10 training on a single machine
import argparse
import json
import os
import portpicker
import sys
import time
module_path=os.path.dirname(os.path.abspath(__file__))
sys.path.append(module_path+'/..')
import tmux_backend
import aws_backend
import util as u
u.install_pdb_handler() # drops into pdb on CTRL+\
parser = argparse.ArgumentParser(description='Launch CIFAR training')
# TODO: rename to gradient instance type
parser.add_argument('--instance-type', type=str, default='g3.4xlarge',
help='instance to use for gradient workers')
parser.add_argument("--num-gpus", default=1, type=int,
help="Number of GPUs to use per worker.")
parser.add_argument('--name', type=str, default='cifar00',
help="name of the current run")
parser.add_argument('--steps', type=int, default=1000,
help="number of steps to run for")
parser.add_argument('--zone', type=str, default='us-east-1c',
help='which availability zone to use')
parser.add_argument('--backend', type=str, default='tmux',
help='tmux or aws')
def main():
u.install_pdb_handler()
u.seed_random(1)
logdir = u.create_local_logdir(args.logdir)
run_name = os.path.basename(logdir)
gl.event_writer = SummaryWriter(logdir)
print(f"Logging to {logdir}")
loss_type = 'CrossEntropy'
d1 = args.data_width ** 2
args.stats_batch_size = min(args.stats_batch_size, args.dataset_size)
args.train_batch_size = min(args.train_batch_size, args.dataset_size)
n = args.stats_batch_size
o = 10
d = [d1, 60, 60, 60, o]
# dataset_size = args.dataset_size
model = u.SimpleFullyConnected2(d, bias=True, nonlin=args.nonlin, last_layer_linear=True)
model = model.to(gl.device)
u.mark_expensive(model.layers[0]) # to stop grad1/hess calculations on this layer
print(model)
try:
if args.wandb:
wandb.init(project='curv_train_tiny', name=run_name, dir='/tmp/wandb.runs')
wandb.tensorboard.patch(tensorboardX=False)
wandb.config['train_batch'] = args.train_batch_size
wandb.config['stats_batch'] = args.stats_batch_size
wandb.config['n'] = n
except Exception as e:
print(f"wandb crash with {e}")
optimizer = torch.optim.SGD(model.parameters(), lr=args.lr, momentum=0.9)
# optimizer = torch.optim.Adam(model.parameters(), lr=0.03) # make 10x smaller for least-squares loss
dataset = u.TinyMNIST(data_width=args.data_width, dataset_size=args.dataset_size, loss_type=loss_type)
train_loader = torch.utils.data.DataLoader(dataset, batch_size=args.train_batch_size, shuffle=False, drop_last=True)
train_iter = u.infinite_iter(train_loader)
stats_loader = torch.utils.data.DataLoader(dataset, batch_size=args.stats_batch_size, shuffle=False, drop_last=True)
stats_iter = u.infinite_iter(stats_loader)
stats_data, stats_targets = next(stats_iter)
test_dataset = u.TinyMNIST(data_width=args.data_width, train=False, dataset_size=args.dataset_size, loss_type=loss_type)
test_batch_size = min(args.dataset_size, 1000)
test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=test_batch_size, shuffle=False, drop_last=True)
test_iter = u.infinite_iter(test_loader)
if loss_type == 'LeastSquares':
loss_fn = u.least_squares
else: # loss_type == 'CrossEntropy':
loss_fn = nn.CrossEntropyLoss()
autograd_lib.add_hooks(model)
gl.reset_global_step()
last_outer = 0
val_losses = []
for step in range(args.stats_steps):
if last_outer:
u.log_scalars({"time/outer": 1000*(time.perf_counter() - last_outer)})
last_outer = time.perf_counter()
with u.timeit("val_loss"):
test_data, test_targets = next(test_iter)
test_output = model(test_data)
val_loss = loss_fn(test_output, test_targets)
print("val_loss", val_loss.item())
val_losses.append(val_loss.item())
u.log_scalar(val_loss=val_loss.item())
with u.timeit("validate"):
if loss_type == 'CrossEntropy':
val_accuracy, val_loss = validate(model, test_loader, f'test (stats_step {step})')
# train_accuracy, train_loss = validate(model, train_loader, f'train (stats_step {step})')
metrics = {'stats_step': step, 'val_accuracy': val_accuracy, 'val_loss': val_loss}
u.log_scalars(metrics)
data, targets = stats_data, stats_targets
if not args.skip_stats:
# Capture Hessian and gradient stats
autograd_lib.enable_hooks()
autograd_lib.clear_backprops(model)
autograd_lib.clear_hess_backprops(model)
with u.timeit("backprop_g"):
output = model(data)
loss = loss_fn(output, targets)
loss.backward(retain_graph=True)
with u.timeit("backprop_H"):
autograd_lib.backprop_hess(output, hess_type=loss_type)
autograd_lib.disable_hooks() # TODO(y): use remove_hooks
with u.timeit("compute_grad1"):
autograd_lib.compute_grad1(model)
with u.timeit("compute_hess"):
autograd_lib.compute_hess(model)
for (i, layer) in enumerate(model.layers):
if hasattr(layer, 'expensive'):
continue
param_names = {layer.weight: "weight", layer.bias: "bias"}
for param in [layer.weight, layer.bias]:
# input/output layers are unreasonably expensive if not using Kronecker factoring
if d[i]*d[i+1] > 8000:
print(f'layer {i} is too big ({d[i],d[i+1]}), skipping stats')
continue
s = AttrDefault(str, {}) # dictionary-like object for layer stats
#############################
# Gradient stats
#############################
A_t = layer.activations
B_t = layer.backprops_list[0] * n
s.sparsity = torch.sum(layer.output <= 0) / layer.output.numel() # proportion of activations that are zero
s.mean_activation = torch.mean(A_t)
s.mean_backprop = torch.mean(B_t)
# empirical Fisher
G = param.grad1.reshape((n, -1))
g = G.mean(dim=0, keepdim=True)
u.nan_check(G)
with u.timeit(f'sigma-{i}'):
efisher = G.t() @ G / n
sigma = efisher - g.t() @ g
# sigma_spectrum =
s.sigma_l2 = u.sym_l2_norm(sigma)
s.sigma_erank = torch.trace(sigma)/s.sigma_l2
H = param.hess
lambda_regularizer = args.lmb * torch.eye(H.shape[0]).to(gl.device)
u.nan_check(H)
with u.timeit(f"invH-{i}"):
invH = torch.cholesky_inverse(H+lambda_regularizer)
with u.timeit(f"H_l2-{i}"):
s.H_l2 = u.sym_l2_norm(H)
s.iH_l2 = u.sym_l2_norm(invH)
with u.timeit(f"norms-{i}"):
s.H_fro = H.flatten().norm()
s.iH_fro = invH.flatten().norm()
s.grad_fro = g.flatten().norm()
s.param_fro = param.data.flatten().norm()
def loss_direction(dd: torch.Tensor, eps):
"""loss improvement if we take step eps in direction dd"""
return u.to_python_scalar(eps * (dd @ g.t()) - 0.5 * eps ** 2 * dd @ H @ dd.t())
def curv_direction(dd: torch.Tensor):
"""Curvature in direction dd"""
return u.to_python_scalar(dd @ H @ dd.t() / (dd.flatten().norm() ** 2))
with u.timeit(f"pinvH-{i}"):
pinvH = u.pinv(H)
with u.timeit(f'curv-{i}'):
s.grad_curv = curv_direction(g) # curvature (eigenvalue) in direction g
ndir = g @ pinvH # newton direction
s.newton_curv = curv_direction(ndir)
setattr(layer.weight, 'pre', pinvH) # save Newton preconditioner
s.step_openai = 1 / s.grad_curv if s.grad_curv else 1234567
s.step_div_inf = 2 / s.H_l2 # divegent step size for batch_size=infinity
s.step_div_1 = torch.tensor(2) / torch.trace(H) # divergent step for batch_size=1
s.newton_fro = ndir.flatten().norm() # frobenius norm of Newton update
s.regret_newton = u.to_python_scalar(g @ pinvH @ g.t() / 2) # replace with "quadratic_form"
s.regret_gradient = loss_direction(g, s.step_openai)
with u.timeit(f'rho-{i}'):
s.rho, s.lyap_erank, lyap_evals = u.truncated_lyapunov_rho(H, sigma)
s.step_div_1_adjusted = s.step_div_1/s.rho
with u.timeit(f"batch-{i}"):
s.batch_openai = torch.trace(H @ sigma) / (g @ H @ g.t())
s.diversity = torch.norm(G, "fro") ** 2 / torch.norm(g) ** 2 / n # Gradient diversity / n
s.noise_variance_pinv = torch.trace(pinvH @ sigma)
s.H_erank = torch.trace(H) / s.H_l2
s.batch_jain_simple = 1 + s.H_erank
s.batch_jain_full = 1 + s.rho * s.H_erank
param_name = f"{layer.name}={param_names[param]}"
u.log_scalars(u.nest_stats(f"{param_name}", s))
H_evals = u.symeig_pos_evals(H)
sigma_evals = u.symeig_pos_evals(sigma)
u.log_spectrum(f'{param_name}/hess', H_evals)
u.log_spectrum(f'{param_name}/sigma', sigma_evals)
u.log_spectrum(f'{param_name}/lyap', lyap_evals)
# gradient steps
with u.timeit('inner'):
for i in range(args.train_steps):
optimizer.zero_grad()
data, targets = next(train_iter)
model.zero_grad()
output = model(data)
loss = loss_fn(output, targets)
loss.backward()
optimizer.step()
if args.weight_decay:
for group in optimizer.param_groups:
for param in group['params']:
param.data.mul_(1-args.weight_decay)
gl.increment_global_step(data.shape[0])
gl.event_writer.close()
def test_factored_stats_golden_values():
"""Test stats from values generated by non-factored version"""
u.seed_random(1)
u.install_pdb_handler()
torch.set_default_dtype(torch.float32)
parser = argparse.ArgumentParser(description='PyTorch MNIST Example')
args = parser.parse_args()
logdir = u.create_local_logdir('/temp/runs/factored_test')
run_name = os.path.basename(logdir)
gl.event_writer = SummaryWriter(logdir)
print('logging to ', logdir)
loss_type = 'LeastSquares'
args.data_width = 2
args.dataset_size = 5
args.stats_batch_size = 5
d1 = args.data_width**2
args.stats_batch_size = args.dataset_size
args.stats_steps = 1
n = args.stats_batch_size
o = 10
d = [d1, o]
model = u.SimpleFullyConnected2(d, bias=False, nonlin=0)
model = model.to(gl.device)
print(model)
dataset = u.TinyMNIST(data_width=args.data_width,
dataset_size=args.dataset_size,
loss_type=loss_type)
stats_loader = torch.utils.data.DataLoader(
dataset, batch_size=args.stats_batch_size, shuffle=False)
stats_iter = u.infinite_iter(stats_loader)
stats_data, stats_targets = next(stats_iter)
if loss_type == 'LeastSquares':
loss_fn = u.least_squares
else: # loss_type == 'CrossEntropy':
loss_fn = nn.CrossEntropyLoss()
autograd_lib.add_hooks(model)
gl.reset_global_step()
last_outer = 0
for step in range(args.stats_steps):
if last_outer:
u.log_scalars(
{"time/outer": 1000 * (time.perf_counter() - last_outer)})
last_outer = time.perf_counter()
data, targets = stats_data, stats_targets
# Capture Hessian and gradient stats
autograd_lib.enable_hooks()
autograd_lib.clear_backprops(model)
with u.timeit("backprop_g"):
output = model(data)
loss = loss_fn(output, targets)
loss.backward(retain_graph=True)
autograd_lib.clear_hess_backprops(model)
with u.timeit("backprop_H"):
autograd_lib.backprop_hess(output, hess_type=loss_type)
autograd_lib.disable_hooks() # TODO(y): use remove_hooks
with u.timeit("compute_grad1"):
autograd_lib.compute_grad1(model)
with u.timeit("compute_hess"):
autograd_lib.compute_hess(model)
autograd_lib.compute_hess(model, method='kron', attr_name='hess2')
autograd_lib.compute_stats_factored(model)
params = list(model.parameters())
assert len(params) == 1
new_values = params[0].stats
golden_values = torch.load('test/factored.pt')
for valname in new_values:
print("Checking ", valname)
if valname == 'sigma_l2':
u.check_close(new_values[valname],
golden_values[valname],
atol=1e-2) # sigma is approximate
elif valname == 'sigma_erank':
u.check_close(new_values[valname],
golden_values[valname],
atol=0.11) # 1.0 vs 1.1
elif valname in ['rho', 'step_div_1_adjusted', 'batch_jain_full']:
continue # lyapunov stats weren't computed correctly in golden set
elif valname in ['batch_openai']:
continue # batch sizes depend on sigma which is approximate
elif valname in ['noise_variance_pinv']:
pass # went from 0.22 to 0.014 after kron factoring (0.01 with full centering, 0.3 with no centering)
elif valname in ['sparsity']:
pass # had a bug in old calc (using integer arithmetic)
else:
u.check_close(new_values[valname],
golden_values[valname],
rtol=1e-4,
atol=1e-6,
label=valname)
gl.event_writer.close()
def main():
u.install_pdb_handler()
u.seed_random(1)
logdir = u.create_local_logdir(args.logdir)
run_name = os.path.basename(logdir)
gl.event_writer = SummaryWriter(logdir)
print(f"Logging to {logdir}")
loss_type = 'CrossEntropy'
d1 = args.data_width ** 2
args.stats_batch_size = min(args.stats_batch_size, args.dataset_size)
args.train_batch_size = min(args.train_batch_size, args.dataset_size)
n = args.stats_batch_size
o = 10
d = [d1, 60, 60, 60, o]
# dataset_size = args.dataset_size
model = u.SimpleFullyConnected2(d, bias=True, nonlin=args.nonlin, last_layer_linear=True)
model = model.to(gl.device)
u.mark_expensive(model.layers[0]) # to stop grad1/hess calculations on this layer
print(model)
try:
if args.wandb:
wandb.init(project='curv_train_tiny', name=run_name, dir='/tmp/wandb.runs')
wandb.tensorboard.patch(tensorboardX=False)
wandb.config['train_batch'] = args.train_batch_size
wandb.config['stats_batch'] = args.stats_batch_size
wandb.config['n'] = n
except Exception as e:
print(f"wandb crash with {e}")
optimizer = torch.optim.SGD(model.parameters(), lr=args.lr, momentum=0.9)
# optimizer = torch.optim.Adam(model.parameters(), lr=0.03) # make 10x smaller for least-squares loss
dataset = u.TinyMNIST(data_width=args.data_width, dataset_size=args.dataset_size, loss_type=loss_type)
train_loader = torch.utils.data.DataLoader(dataset, batch_size=args.train_batch_size, shuffle=False, drop_last=True)
train_iter = u.infinite_iter(train_loader)
stats_loader = torch.utils.data.DataLoader(dataset, batch_size=args.stats_batch_size, shuffle=False, drop_last=True)
stats_iter = u.infinite_iter(stats_loader)
stats_data, stats_targets = next(stats_iter)
test_dataset = u.TinyMNIST(data_width=args.data_width, train=False, dataset_size=args.dataset_size, loss_type=loss_type)
test_batch_size = min(args.dataset_size, 1000)
test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=test_batch_size, shuffle=False, drop_last=True)
test_iter = u.infinite_iter(test_loader)
if loss_type == 'LeastSquares':
loss_fn = u.least_squares
else: # loss_type == 'CrossEntropy':
loss_fn = nn.CrossEntropyLoss()
autograd_lib.add_hooks(model)
gl.reset_global_step()
last_outer = 0
val_losses = []
for step in range(args.stats_steps):
if last_outer:
u.log_scalars({"time/outer": 1000*(time.perf_counter() - last_outer)})
last_outer = time.perf_counter()
with u.timeit("val_loss"):
test_data, test_targets = next(test_iter)
test_output = model(test_data)
val_loss = loss_fn(test_output, test_targets)
print("val_loss", val_loss.item())
val_losses.append(val_loss.item())
u.log_scalar(val_loss=val_loss.item())
with u.timeit("validate"):
if loss_type == 'CrossEntropy':
val_accuracy, val_loss = validate(model, test_loader, f'test (stats_step {step})')
# train_accuracy, train_loss = validate(model, train_loader, f'train (stats_step {step})')
metrics = {'stats_step': step, 'val_accuracy': val_accuracy, 'val_loss': val_loss}
u.log_scalars(metrics)
data, targets = stats_data, stats_targets
if not args.skip_stats:
# Capture Hessian and gradient stats
autograd_lib.enable_hooks()
autograd_lib.clear_backprops(model)
autograd_lib.clear_hess_backprops(model)
with u.timeit("backprop_g"):
output = model(data)
loss = loss_fn(output, targets)
loss.backward(retain_graph=True)
with u.timeit("backprop_H"):
autograd_lib.backprop_hess(output, hess_type='CrossEntropy')
autograd_lib.disable_hooks() # TODO(y): use remove_hooks
with u.timeit("compute_grad1"):
autograd_lib.compute_grad1(model)
with u.timeit("compute_hess"):
autograd_lib.compute_hess(model, method='kron', attr_name='hess2')
autograd_lib.compute_stats_factored(model)
for (i, layer) in enumerate(model.layers):
param_names = {layer.weight: "weight", layer.bias: "bias"}
for param in [layer.weight, layer.bias]:
if param is None:
continue
if not hasattr(param, 'stats'):
continue
s = param.stats
param_name = param_names[param]
u.log_scalars(u.nest_stats(f"{param_name}", s))
# gradient steps
with u.timeit('inner'):
for i in range(args.train_steps):
optimizer.zero_grad()
data, targets = next(train_iter)
model.zero_grad()
output = model(data)
loss = loss_fn(output, targets)
loss.backward()
optimizer.step()
if args.weight_decay:
for group in optimizer.param_groups:
for param in group['params']:
param.data.mul_(1-args.weight_decay)
gl.increment_global_step(data.shape[0])
gl.event_writer.close()