def train(model,
X_train,
Y_train,
optim,
steps,
BS=128,
device=Device.CPU,
lossfn=sparse_categorical_crossentropy):
Tensor.training = True
if device == Device.GPU: [x.gpu_() for x in get_parameters([model, optim])]
elif device == Device.ANE:
[x.ane_() for x in get_parameters([model, optim])]
losses, accuracies = [], []
for i in (t := trange(steps, disable=os.getenv('CI') is not None)):
samp = np.random.randint(0, X_train.shape[0], size=(BS))
x = Tensor(X_train[samp], device=device)
y = Y_train[samp]
# network
out = model.forward(x)
loss = lossfn(out, y)
optim.zero_grad()
loss.backward()
optim.step()
cat = np.argmax(out.cpu().data, axis=-1)
accuracy = (cat == y).mean()
# printing
loss = loss.cpu().data
losses.append(loss)
accuracies.append(accuracy)
t.set_description("loss %.2f accuracy %.2f" % (loss, accuracy))
def step_tinygrad(optim, kwargs={}, device=Device.CPU):
net = TinyNet()
optim = optim([net.x, net.W], **kwargs)
if device == Device.GPU: [x.gpu_() for x in get_parameters([net, optim])]
elif device == Device.ANE: [x.ane_() for x in get_parameters([net, optim])]
out = net.forward()
out.backward()
optim.step()
return net.x.cpu().data, net.W.cpu().data
def parameters(self):
if DEBUG: #keeping this for a moment
pars = [par for par in get_parameters(self) if par.requires_grad]
no_pars = 0
for par in pars:
print(par.shape)
no_pars += np.prod(par.shape)
print('no of parameters', no_pars)
return pars
else:
return get_parameters(self)
def train(model, optim, steps, BS=128, gpu=False):
if gpu is True: [x.cuda_() for x in get_parameters([model, optim])]
losses, accuracies = [], []
for i in (t := trange(steps, disable=os.getenv('CI') is not None)):
samp = np.random.randint(0, X_train.shape[0], size=(BS))
x = Tensor(X_train[samp].reshape((-1, 28 * 28)).astype(np.float32),
gpu=gpu)
Y = Y_train[samp]
y = np.zeros((len(samp), 10), np.float32)
# correct loss for NLL, torch NLL loss returns one per row
y[range(y.shape[0]), Y] = -10.0
y = Tensor(y, gpu=gpu)
# network
out = model.forward(x)
# NLL loss function
loss = out.mul(y).mean()
optim.zero_grad()
loss.backward()
optim.step()
cat = np.argmax(out.cpu().data, axis=1)
accuracy = (cat == Y).mean()
# printing
loss = loss.cpu().data
losses.append(loss)
accuracies.append(accuracy)
t.set_description("loss %.2f accuracy %.2f" % (loss, accuracy))
def step_tinygrad(optim, kwargs={}, gpu=False):
net = TinyNet()
optim = optim([net.x, net.W], **kwargs)
if gpu is True: [x.cuda_() for x in get_parameters([net, optim])]
out = net.forward()
out.backward()
optim.step()
return net.x.cpu().data, net.W.cpu().data
def load(self, filename):
with open(filename+'.npy', 'rb') as f:
for par in get_parameters(self):
#if par.requires_grad:
try:
par.cpu().data[:] = np.load(f)
if GPU:
par.cuda()
except:
print('Could not load parameter')
def train(model,
X_train,
Y_train,
optim,
steps,
num_classes=None,
BS=128,
device=Device.CPU,
lossfn=lambda out, y: out.mul(y).mean()):
if device == Device.GPU: [x.gpu_() for x in get_parameters([model, optim])]
elif device == Device.ANE:
[x.ane_() for x in get_parameters([model, optim])]
if num_classes is None: num_classes = Y_train.max().astype(int) + 1
losses, accuracies = [], []
for i in (t := trange(steps, disable=os.getenv('CI') is not None)):
samp = np.random.randint(0, X_train.shape[0], size=(BS))
x = Tensor(X_train[samp].reshape((-1, 28 * 28)).astype(np.float32),
device=device)
Y = Y_train[samp]
y = np.zeros((len(samp), num_classes), np.float32)
# correct loss for NLL, torch NLL loss returns one per row
y[range(y.shape[0]), Y] = -1.0 * num_classes
y = Tensor(y, device=device)
# network
out = model.forward(x)
# NLL loss function
loss = lossfn(out, y)
optim.zero_grad()
loss.backward()
optim.step()
cat = np.argmax(out.cpu().data, axis=1)
accuracy = (cat == Y).mean()
# printing
loss = loss.cpu().data
losses.append(loss)
accuracies.append(accuracy)
t.set_description("loss %.2f accuracy %.2f" % (loss, accuracy))
def train_one_step(model,X,Y):
params = get_parameters(model)
pcount = 0
for p in params:
pcount += np.prod(p.shape)
optimizer = optim.Adam(params, lr=0.001)
print("stepping %r with %.1fM params bs %d" % (type(model), pcount/1e6, BS))
st = time.time()
train(model, X, Y, optimizer, steps=1, BS=BS)
et = time.time()-st
print("done in %.2f ms" % (et*1000.))
def parameters(self): return get_parameters(self)
def save(self, filename):
with open(filename+'.npy', 'wb') as f:
for par in get_parameters(self):
#if par.requires_grad:
np.save(f, par.cpu().data)
lossfn = lambda out,y: out.mul(y).mean() + lmbd*(model.weight1.abs() + model.weight2.abs()).sum()
X_train, Y_train, X_test, Y_test = fetch_mnist()
steps = len(X_train)//BS
np.random.seed(1337)
if QUICK:
steps = 1
X_test, Y_test = X_test[:BS], Y_test[:BS]
model = BigConvNet()
if len(sys.argv) > 1:
try:
model.load(sys.argv[1])
print('Loaded weights "'+sys.argv[1]+'", evaluating...')
evaluate(model, X_test, Y_test, BS=BS)
except:
print('could not load weights "'+sys.argv[1]+'".')
if GPU:
params = get_parameters(model)
[x.cuda_() for x in params]
for lr, epochs in zip(lrs, epochss):
optimizer = optim.Adam(model.parameters(), lr=lr)
for epoch in range(1,epochs+1):
#first epoch without augmentation
X_aug = X_train if epoch == 1 else augment_img(X_train)
train(model, X_aug, Y_train, optimizer, steps=steps, lossfn=lossfn, gpu=GPU, BS=BS)
accuracy = evaluate(model, X_test, Y_test, BS=BS)
model.save('examples/checkpoint'+str("%.0f" % (accuracy*1.0e6)))
ds_X = ds[:, 0:6]
ds_Y = np.copy(ds[:, 1:])
ds_X_train, ds_X_test = ds_X[0:8000], ds_X[8000:]
ds_Y_train, ds_Y_test = ds_Y[0:8000], ds_Y[8000:]
return ds_X_train, ds_Y_train, ds_X_test, ds_Y_test
from tinygrad.optim import Adam
if __name__ == "__main__":
model = Transformer(10, 6, 2, 128, 4)
X_train, Y_train, X_test, Y_test = make_dataset()
lr = 0.003
for i in range(10):
optim = Adam(get_parameters(model), lr=lr)
train(model, X_train, Y_train, optim, 50, BS=64)
acc, Y_test_preds = evaluate(model,
X_test,
Y_test,
num_classes=10,
return_predict=True)
lr /= 1.2
print(f'reducing lr to {lr:.4f}')
if acc > 0.998:
wrong = 0
for k in range(len(Y_test_preds)):
if (Y_test_preds[k] != Y_test[k]).any():
wrong += 1
a, b, c, x = X_test[k, :2], X_test[k, 2:4], Y_test[
k, -3:], Y_test_preds[k, -3:]
def forward(self, x):
bs = x.shape[0]
xnp = x.cpu().data
onehot = np.zeros((bs, x.shape[1], self.maxlen + self.syms),
dtype=np.float32)
for i in range(x.shape[1]):
onehot[range(bs), i, i] = 1
onehot[range(bs), i, self.maxlen + xnp[:, i]] = 1
onehot = onehot.reshape(bs * x.shape[1], self.maxlen + self.syms)
x = Tensor(onehot, device=x.device).dot(
self.embed).reshape(shape=(bs, x.shape[1], -1))
for t in self.tbs:
x = t(x)
x = x.reshape(shape=(-1, x.shape[-1])).dot(self.final).logsoftmax()
return x.reshape(shape=(bs, -1, x.shape[-1]))
from tinygrad.optim import Adam
if __name__ == "__main__":
model = Transformer(10, 6, 2, 128, 4)
#in1 = Tensor.zeros(20, 6, 128)
#ret = model.forward(in1)
#print(ret.shape)
X_train, Y_train, X_test, Y_test = make_dataset()
optim = Adam(get_parameters(model), lr=0.001)
train(model, X_train, Y_train, optim, 100)
# if you want to look at the outputs
"""
import matplotlib.pyplot as plt
plt.plot(out.data[0])
plt.show()
"""
return out, retimg
if __name__ == "__main__":
# instantiate my net
model = EfficientNet(int(os.getenv("NUM", "0")))
model.load_weights_from_torch()
if GPU:
[x.gpu_() for x in get_parameters(model)]
# category labels
import ast
lbls = fetch(
"https://gist.githubusercontent.com/yrevar/942d3a0ac09ec9e5eb3a/raw/238f720ff059c1f82f368259d1ca4ffa5dd8f9f5/imagenet1000_clsidx_to_labels.txt"
)
lbls = ast.literal_eval(lbls.decode('utf-8'))
# load image and preprocess
from PIL import Image
url = sys.argv[1]
if url == 'webcam':
import cv2
cap = cv2.VideoCapture(0)
cap.set(cv2.CAP_PROP_BUFFERSIZE, 1)
# if you want to look at the outputs
"""
import matplotlib.pyplot as plt
plt.plot(out.data[0])
plt.show()
"""
return out, retimg
if __name__ == "__main__":
# instantiate my net
model = EfficientNet(int(os.getenv("NUM", "0")))
model.load_weights_from_torch()
if GPU:
[x.cuda_() for x in get_parameters(model)]
# category labels
import ast
lbls = fetch(
"https://gist.githubusercontent.com/yrevar/942d3a0ac09ec9e5eb3a/raw/238f720ff059c1f82f368259d1ca4ffa5dd8f9f5/imagenet1000_clsidx_to_labels.txt"
)
lbls = ast.literal_eval(lbls.decode('utf-8'))
# load image and preprocess
from PIL import Image
url = sys.argv[1]
if url == 'webcam':
import cv2
cap = cv2.VideoCapture(0)
cap.set(cv2.CAP_PROP_BUFFERSIZE, 1)
x = x.dot(self.l2).leakyrelu(0.2)
if train:
x = x.dropout(0.3)
x = x.dot(self.l3).leakyrelu(0.2)
if train:
x = x.dropout(0.3)
x = x.dot(self.l4).logsoftmax()
return x
if __name__ == "__main__":
generator = LinearGen()
discriminator = LinearDisc()
batch_size = 128
k = 1
epochs = 100
generator_params = get_parameters(generator)
discriminator_params = get_parameters(discriminator)
gen_loss = []
disc_loss = []
output_folder = "outputs"
os.makedirs(output_folder, exist_ok=True)
train_data_size = len(X_train)
ds_noise = Tensor(np.random.uniform(size=(64,128)).astype(np.float32), gpu=GPU, requires_grad=False)
n_steps = int(train_data_size/batch_size)
if GPU:
[x.cuda_() for x in generator_params+discriminator_params]
# optimizers
optim_g = optim.Adam(generator_params, lr=0.001)
optim_d = optim.Adam(discriminator_params, lr=0.001)
def regularization_l2(model, a=1e-4):
