def train_network(model, num_epochs = 100, minibatch_size=256, dropout_rate = 0.5, eps_w = 0.01, eps_b = 0.01, mom = 0.9, wd = 0.0005): num_layers = model.num_layers num_weights = model.num_weights last = time.time() num_samples = minibatch_size minibatch_size = minibatch_size / num_gpu for i in xrange(num_epochs): print "---------------------Epoch #", i weightsgrad = [[None] * num_weights for z in range(num_gpu)] biasgrad = [[None] * num_weights for z in range(num_gpu)] for j in xrange(1, 1024): count = j % num_gpu owl.set_device(gpu_array[count]) data = owl.randn([227, 227, 3, minibatch_size], 0, 1) label = owl.randn([1, minibatch_size], 0, 1) out = train_one_mb(model, data, label, weightsgrad[count], biasgrad[count], dropout_rate) out.start_eval() if count == 0: # Update for k in range(num_weights): for l in range(1, num_gpu): weightsgrad[0][k] = weightsgrad[0][k] + weightsgrad[l][k] biasgrad[0][k] = biasgrad[0][k] + biasgrad[l][k] model.weightsdelta[k] = mom * model.weightsdelta[k] - eps_w / num_samples * (weightsgrad[0][k] + wd * model.weights[k]) model.biasdelta[k] = mom * model.biasdelta[k] - eps_b / num_samples * (biasgrad[0][k] + wd * model.bias[k]) model.weights[k] += model.weightsdelta[k] model.bias[k] += model.biasdelta[k] if j % (lazy * num_gpu) == 0: print_training_accuracy(out, label, minibatch_size) print "time: %s" % (time.time() - last) last = time.time()
def init_random(self): self.weights = [ owl.randn([self.filtersizes[0], self.filtersizes[0], 1, self.filters[0]], 0.0, 0.1), owl.randn([self.filtersizes[1], self.filtersizes[1], self.filters[0], self.filters[1]], 0.0, 0.1), owl.randn([128, self.convolution_output_size], 0.0, 0.1), owl.randn([10, 128], 0.0, 0.1) ]; self.weightdelta = [ owl.zeros([self.filtersizes[0], self.filtersizes[0], 1, self.filters[0]]), owl.zeros([self.filtersizes[1], self.filtersizes[1], self.filters[0], self.filters[1]]), owl.zeros([128, self.convolution_output_size]), owl.zeros([10, 128]) ]; self.bias = [ owl.zeros([self.filters[0]]), owl.zeros([self.filters[1]]), owl.zeros([128, 1]), owl.zeros([10, 1]) ]; self.biasdelta = [ owl.zeros([self.filters[0]]), owl.zeros([self.filters[1]]), owl.zeros([128, 1]), owl.zeros([10, 1]) ];
def train_network(model, num_epochs = 100, minibatch_size=256, dropout_rate = 0.5, eps_w = 0.01, eps_b = 0.01, mom = 0.9, wd = 0.0005): num_layers = model.num_layers num_weights = model.num_weights last = time.time() minibatch_size = minibatch_size # / num_gpu for i in xrange(num_epochs): print "---------------------Epoch #", i weightsgrad = [[None] * num_weights for z in range(num_gpu)] biasgrad = [[None] * num_weights for z in range(num_gpu)] for j in xrange(1, 1024): count = j % num_gpu owl.set_device(gpu_array[count]) data = owl.randn([227, 227, 3, minibatch_size], 0, 1) label = owl.randn([1, minibatch_size], 0, 1) out = train_one_mb(model, data, label, weightsgrad[count], biasgrad[count], dropout_rate) for k in weightsgrad[count]: k.start_eval() for k in biasgrad[count]: k.start_eval() if count == 0: for k in range(0, num_gpu): for l in weightsgrad[k]: l.wait_for_eval() for l in biasgrad[k]: l.wait_for_eval() print "time: %s" % (time.time() - last) last = time.time()
def train_network(model, num_epochs=100, minibatch_size=256, dropout_rate=0.5, eps_w=0.01, eps_b=0.01, mom=0.9, wd=0.0005): gpu0 = owl.create_gpu_device(0) gpu1 = owl.create_gpu_device(1) num_layers = 20 num_weights = 8 count = 0 last = time.time() # dp = ImageNetDataProvider(mean_file='/home/minjie/data/imagenet/imagenet_mean.binaryproto', # train_db='/home/minjie/data/imagenet/ilsvrc12_train_lmdb', # val_db='/home/minjie/data/imagenet/ilsvrc12_val_lmdb', # test_db='/home/minjie/data/imagenet/ilsvrc12_test_lmdb') for i in xrange(num_epochs): print "---------------------Epoch #", i for j in xrange(300): count = count + 1 data = owl.randn([227, 227, 3, minibatch_size], 0, 1) label = owl.randn([1, minibatch_size], 0, 1) weightsgrad = [None] * num_weights biasgrad = [None] * num_weights num_samples = minibatch_size ''' thisimg = samples[0, :] print thisimg imgdata = np.transpose(thisimg.reshape([3, 227*227])).reshape([227, 227, 3]) print imgdata img = Image.fromarray(imgdata.astype(np.uint8)) img.save('testimg.jpg', format='JPEG') exit(0) ''' owl.set_device(gpu0) out = train_one_mb(model, data, label, weightsgrad, biasgrad, dropout_rate) for k in range(num_weights): model.weightsdelta[ k] = mom * model.weightsdelta[k] - eps_w / num_samples * ( weightsgrad[k] + wd * model.weights[k]) model.biasdelta[ k] = mom * model.biasdelta[k] - eps_b / num_samples * ( biasgrad[k] + wd * model.bias[k]) model.weights[k] += model.weightsdelta[k] model.weights[k].start_eval() model.bias[k] += model.biasdelta[k] model.bias[k].start_eval() if count % 3 == 0: print_training_accuracy(out, label, data.shape[-1]) print "time: %s" % (time.time() - last) last = time.time()
def __init__(self, input_size, hidden_size, output_size): self.Layers = [input_size, hidden_size, output_size] # Recurrent weights: take x_t, h_{t-1}, and bias unit # and produce the 3 gates and the input to cell signal self.ig_weight_data = owl.randn([self.Layers[0], self.Layers[1]], 0.0, 0.1) self.fg_weight_data = owl.randn([self.Layers[0], self.Layers[1]], 0.0, 0.1) self.og_weight_data = owl.randn([self.Layers[0], self.Layers[1]], 0.0, 0.1) self.ff_weight_data = owl.randn([self.Layers[0], self.Layers[1]], 0.0, 0.1) self.ig_weight_prev = owl.randn([self.Layers[1], self.Layers[1]], 0.0, 0.1) self.fg_weight_prev = owl.randn([self.Layers[1], self.Layers[1]], 0.0, 0.1) self.og_weight_prev = owl.randn([self.Layers[1], self.Layers[1]], 0.0, 0.1) self.ff_weight_prev = owl.randn([self.Layers[1], self.Layers[1]], 0.0, 0.1) self.ig_weight_bias = owl.zeros([self.Layers[1], 1]) self.fg_weight_bias = owl.zeros([self.Layers[1], 1]) self.og_weight_bias = owl.zeros([self.Layers[1], 1]) self.ff_weight_bias = owl.zeros([self.Layers[1], 1]) # Decoder weights (e.g. mapping to vocabulary) self.decoder_weights = owl.randn([self.Layers[1], self.Layers[2]], 0.0, 0.1) # decoder self.decoder_bias = owl.zeros([output_size, 1])
def train_network(model, num_epochs = 100, minibatch_size=256, dropout_rate = 0.5, eps_w = 0.01, eps_b = 0.01, mom = 0.9, wd = 0.0005): gpu0 = owl.create_gpu_device(0) gpu1 = owl.create_gpu_device(1) num_layers = 20 num_weights = 8 count = 0 last = time.time() # dp = ImageNetDataProvider(mean_file='/home/minjie/data/imagenet/imagenet_mean.binaryproto', # train_db='/home/minjie/data/imagenet/ilsvrc12_train_lmdb', # val_db='/home/minjie/data/imagenet/ilsvrc12_val_lmdb', # test_db='/home/minjie/data/imagenet/ilsvrc12_test_lmdb') for i in xrange(num_epochs): print "---------------------Epoch #", i for j in xrange(300): count = count + 1 data = owl.randn([227, 227, 3, minibatch_size], 0, 1) label = owl.randn([1, minibatch_size], 0, 1) weightsgrad = [None] * num_weights biasgrad = [None] * num_weights num_samples = minibatch_size ''' thisimg = samples[0, :] print thisimg imgdata = np.transpose(thisimg.reshape([3, 227*227])).reshape([227, 227, 3]) print imgdata img = Image.fromarray(imgdata.astype(np.uint8)) img.save('testimg.jpg', format='JPEG') exit(0) ''' owl.set_device(gpu0) out = train_one_mb(model, data, label, weightsgrad, biasgrad, dropout_rate) for k in range(num_weights): model.weightsdelta[k] = mom * model.weightsdelta[k] - eps_w / num_samples * (weightsgrad[k] + wd * model.weights[k]) model.biasdelta[k] = mom * model.biasdelta[k] - eps_b / num_samples * (biasgrad[k] + wd * model.bias[k]) model.weights[k] += model.weightsdelta[k] model.weights[k].start_eval() model.bias[k] += model.biasdelta[k] model.bias[k].start_eval() if count % 3 == 0: print_training_accuracy(out, label, data.shape[-1]) print "time: %s" % (time.time() - last) last = time.time()
def __init__(self, data_file='mnist_all.mat', num_epochs=100, mb_size=256, eps_w=0.01, eps_b=0.01): self.cpu = owl.create_cpu_device() self.gpu = owl.create_gpu_device(0) self.data_file = data_file self.num_epochs=num_epochs self.mb_size=mb_size self.eps_w=eps_w self.eps_b=eps_b # init weight l1 = 784; l2 = 256; l3 = 10 self.l1 = l1; self.l2 = l2; self.l3 = l3 self.w1 = owl.randn([l2, l1], 0.0, math.sqrt(4.0 / (l1 + l2))) self.w2 = owl.randn([l3, l2], 0.0, math.sqrt(4.0 / (l2 + l3))) self.b1 = owl.zeros([l2, 1]) self.b2 = owl.zeros([l3, 1])
def init_random(self): self.weights = [ owl.randn([5, 5, 1, 16], 0.0, 0.1), owl.randn([5, 5, 16, 32], 0.0, 0.1), owl.randn([10, 512], 0.0, 0.1) ]; self.weightdelta = [ owl.zeros([5, 5, 1, 16]), owl.zeros([5, 5, 16, 32]), owl.zeros([10, 512]) ]; self.bias = self.biasdelta = [ owl.zeros([16]), owl.zeros([32]), owl.zeros([10, 1]) ];
def init_random(self): self.weights = [ owl.randn([5, 5, 1, 16], 0.0, 0.1), owl.randn([5, 5, 16, 32], 0.0, 0.1), owl.randn([10, 512], 0.0, 0.1) ] self.weightdelta = [ owl.zeros([5, 5, 1, 16]), owl.zeros([5, 5, 16, 32]), owl.zeros([10, 512]) ] self.bias = [owl.zeros([16]), owl.zeros([32]), owl.zeros([10, 1])] self.biasdelta = [ owl.zeros([16]), owl.zeros([32]), owl.zeros([10, 1]) ]
def init_random(self): self.weights = [ owl.randn([11, 11, 3, 96], 0.0, 0.01), owl.randn([5, 5, 96, 256], 0.0, 0.01), owl.randn([3, 3, 256, 384], 0.0, 0.01), owl.randn([3, 3, 384, 384], 0.0, 0.01), owl.randn([3, 3, 384, 256], 0.0, 0.01), owl.randn([4096, 9216], 0.0, 0.01), owl.randn([4096, 4096], 0.0, 0.01), owl.randn([1000, 4096], 0.0, 0.01) ]; self.weightsdelta = [ owl.zeros([11, 11, 3, 96]), owl.zeros([5, 5, 96, 256]), owl.zeros([3, 3, 256, 384]), owl.zeros([3, 3, 384, 384]), owl.zeros([3, 3, 384, 256]), owl.zeros([4096, 9216]), owl.zeros([4096, 4096]), owl.zeros([1000, 4096]) ]; self.bias = [ owl.zeros([96]), owl.zeros([256]), owl.zeros([384]), owl.zeros([384]), owl.zeros([256]), owl.zeros([4096, 1]), owl.zeros([4096, 1]), owl.zeros([1000, 1]) ]; self.biasdelta = [ owl.zeros([96]), owl.zeros([256]), owl.zeros([384]), owl.zeros([384]), owl.zeros([256]), owl.zeros([4096, 1]), owl.zeros([4096, 1]), owl.zeros([1000, 1]) ];
def __init__(self, vocab_size, input_size, hidden_size): output_size = vocab_size self.Layers = [input_size, hidden_size, output_size] print 'Model size:', self.Layers # Recurrent weights: take x_t, h_{t-1}, and bias unit # and produce the 3 gates and the input to cell signal # self.WIFOG = owl.randn([self.Layers[0] + self.Layers[1], self.Layers[1] * 4], 0.0, 0.1) # self.BIFOG = owl.zeros([self.Layers[1] * 4, 1]) self.ig_weight_data = owl.randn([self.Layers[1], self.Layers[0]], 0.0, 0.1) self.fg_weight_data = owl.randn([self.Layers[1], self.Layers[0]], 0.0, 0.1) self.og_weight_data = owl.randn([self.Layers[1], self.Layers[0]], 0.0, 0.1) self.ff_weight_data = owl.randn([self.Layers[1], self.Layers[0]], 0.0, 0.1) self.ig_weight_prev = owl.randn([self.Layers[1], self.Layers[1]], 0.0, 0.1) self.fg_weight_prev = owl.randn([self.Layers[1], self.Layers[1]], 0.0, 0.1) self.og_weight_prev = owl.randn([self.Layers[1], self.Layers[1]], 0.0, 0.1) self.ff_weight_prev = owl.randn([self.Layers[1], self.Layers[1]], 0.0, 0.1) self.ig_weight_cell = owl.randn([self.Layers[1], self.Layers[1]], 0.0, 0.1) self.fg_weight_cell = owl.randn([self.Layers[1], self.Layers[1]], 0.0, 0.1) self.og_weight_cell = owl.randn([self.Layers[1], self.Layers[1]], 0.0, 0.1) self.ff_weight_cell = owl.randn([self.Layers[1], self.Layers[1]], 0.0, 0.1) self.ig_weight_bias = owl.zeros([self.Layers[1], 1]) self.fg_weight_bias = owl.zeros([self.Layers[1], 1]) self.og_weight_bias = owl.zeros([self.Layers[1], 1]) self.ff_weight_bias = owl.zeros([self.Layers[1], 1]) # Decoder weights (e.g. mapping to vocabulary) self.decoder_weights = owl.randn([self.Layers[2], self.Layers[1]], 0.0, 0.1) # decoder self.decoder_bias = owl.zeros([output_size, 1]) self.emb_weight = [None] * vocab_size for i in range(vocab_size): self.emb_weight[i] = owl.randn([input_size, 1], 0.0, 0.1)
import owl import numpy as np import demo_common as dc x1 = owl.randn([784, 128], 0.0, 0.1) x2 = owl.randn([784, 128], 0.0, 0.1) w = owl.randn([512, 784], 0.0, 0.1) b = owl.zeros([512, 1]) y1 = w * x1 + b y2 = w * x2 + b gw = y1 * x1.trans() + y2 * x2.trans() print gw.to_numpy()
import owl import owl.elewise as ele import numpy as np import demo_common x = owl.randn([784, 256], 0.0, 0.01) w = owl.randn([512, 784], 0.0, 0.01) b = owl.zeros([512, 1]) y = ele.relu(w * x + b) print y.to_numpy() e = owl.randn([512, 256], 0.0, 0.01) ey = ele.relu_back(e, y) ex = w.trans() * ey print ex.to_numpy()
data = data / np.var(data, 0) # training parameters epsilon = 0.01 momentum = 0.9 num_epochs = 40 batch_size = 64 num_batches = data.shape[0]//batch_size # model parameters num_vis = data.shape[1] num_hid = 1024 # initialize weights weights = 0.1 * owl.randn([num_vis, num_hid],0,1) bias_v = owl.zeros([1,num_vis]) bias_h = owl.zeros([1,num_hid]) # initialize weight updates d_weights = owl.zeros((num_vis,num_hid )) d_bias_v = owl.zeros([1,num_vis]) d_bias_h = owl.zeros([1,num_hid]) start_time = time.time() for epoch in range(num_epochs): print("Epoch %i" % (epoch + 1)) err = [] for batch in range(num_batches): np_set = data[batch*batch_size:(batch + 1)*batch_size,:]
import owl import owl.conv as co import numpy as np import demo_common x = owl.randn([227, 227, 3, 256], 0.0, 1) w = owl.randn([11, 11, 3, 96], 0.0, 0.1) b = owl.zeros([96]) conv = co.Convolver(pad_h=0, pad_w=0, stride_v=4, stride_h=4) y = conv.ff(x, w, b) print y.to_numpy() print y.shape ex = conv.bp(y, w) print ex.to_numpy() print ex.shape
def init_random(self): self.weights = [ owl.randn([3, 3, 3, 64], 0.0, 0.01), owl.randn([3, 3, 64, 64], 0.0, 0.01), owl.randn([1, 1, 1, 1], 0.0, 0.01), owl.randn([3, 3, 64, 128], 0.0, 0.01), owl.randn([3, 3, 128, 128], 0.0, 0.01), owl.randn([1, 1, 1, 1], 0.0, 0.01), owl.randn([3, 3, 128, 256], 0.0, 0.01), owl.randn([3, 3, 256, 256], 0.0, 0.01), owl.randn([3, 3, 256, 256], 0.0, 0.01), owl.randn([1, 1, 1, 1], 0.0, 0.01), owl.randn([3, 3, 256, 512], 0.0, 0.01), owl.randn([3, 3, 512, 512], 0.0, 0.01), owl.randn([3, 3, 512, 512], 0.0, 0.01), owl.randn([1, 1, 1, 1], 0.0, 0.01), owl.randn([3, 3, 512, 512], 0.0, 0.01), owl.randn([3, 3, 512, 512], 0.0, 0.01), owl.randn([3, 3, 512, 512], 0.0, 0.01), owl.randn([1, 1, 1, 1], 0.0, 0.01), owl.randn([4096, 25088], 0.0, 0.005), owl.randn([4096, 4096], 0.0, 0.005), owl.randn([1000, 4096], 0.0, 0.01) ] self.weightsdelta = [ owl.zeros([3, 3, 3, 64]), owl.zeros([3, 3, 64, 64]), owl.zeros([1, 1, 1, 1]), owl.zeros([3, 3, 64, 128]), owl.zeros([3, 3, 128, 128]), owl.zeros([1, 1, 1, 1]), owl.zeros([3, 3, 128, 256]), owl.zeros([3, 3, 256, 256]), owl.zeros([3, 3, 256, 256]), owl.zeros([1, 1, 1, 1]), owl.zeros([3, 3, 256, 512]), owl.zeros([3, 3, 512, 512]), owl.zeros([3, 3, 512, 512]), owl.zeros([1, 1, 1, 1]), owl.zeros([3, 3, 512, 512]), owl.zeros([3, 3, 512, 512]), owl.zeros([3, 3, 512, 512]), owl.zeros([1, 1, 1, 1]), owl.zeros([4096, 25088]), owl.zeros([4096, 4096]), owl.zeros([1000, 4096]) ] self.bias = [ owl.zeros([64]), owl.zeros([64]), owl.zeros([64]), owl.zeros([128]), owl.zeros([128]), owl.zeros([128]), owl.zeros([256]), owl.zeros([256]), owl.zeros([256]), owl.zeros([256]), owl.zeros([512]), owl.zeros([512]), owl.zeros([512]), owl.zeros([512]), owl.zeros([512]), owl.zeros([512]), owl.zeros([512]), owl.zeros([512]), owl.zeros([4096, 1]), owl.zeros([4096, 1]), owl.zeros([1000, 1]) ] self.biasdelta = [ owl.zeros([64]), owl.zeros([64]), owl.zeros([64]), owl.zeros([128]), owl.zeros([128]), owl.zeros([128]), owl.zeros([256]), owl.zeros([256]), owl.zeros([256]), owl.zeros([256]), owl.zeros([512]), owl.zeros([512]), owl.zeros([512]), owl.zeros([512]), owl.zeros([512]), owl.zeros([512]), owl.zeros([512]), owl.zeros([512]), owl.zeros([4096, 1]), owl.zeros([4096, 1]), owl.zeros([1000, 1]) ]
def init_random(self): last_channel = self.input_channel last_scale = self.input_size last_dim = last_scale * last_scale * last_channel for i in range(self.num_weights): if self.ff_infos[i]['ff_type'] == 'conv': kernelsize = self.ff_infos[i]['convolution_param'].kernel_size out_channel = self.ff_infos[i]['convolution_param'].num_output stride = self.ff_infos[i]['convolution_param'].stride pad = self.ff_infos[i]['convolution_param'].pad print 'conv %d %d %d %d %d %d %d %d' % (i, kernelsize, out_channel, stride, pad, last_channel, last_scale, last_dim) owl.randn([kernelsize, kernelsize, last_channel, out_channel], 0.0, self.ff_infos[i]['convolution_param'].weight_filler.std) #weight if self.ff_infos[i]['convolution_param'].weight_filler.type == "gaussian": self.weights.append(owl.randn([kernelsize, kernelsize, last_channel, out_channel], 0.0, self.ff_infos[i]['convolution_param'].weight_filler.std)) elif self.ff_infos[i]['convolution_param'].weight_filler.type == "constant": self.weights.append(owl.zeros([kernelsize, kernelsize, last_channel, out_channel]) + self.ff_infos[i]['convolution_param'].weight_filler.value) else: assert False self.weightsdelta.append(owl.zeros([kernelsize, kernelsize, last_channel, out_channel])) #bias if self.ff_infos[i]['convolution_param'].bias_filler.type == "gaussian": self.bias.append(owl.randn([out_channel], 0.0, self.ff_infos[i]['convolution_param'].bias_filler.std)) elif self.ff_infos[i]['convolution_param'].bias_filler.type == "constant": self.bias.append(owl.zeros([out_channel]) + self.ff_infos[i]['convolution_param'].bias_filler.value) else: assert False self.biasdelta.append(owl.zeros([out_channel])) last_channel = out_channel last_scale = (last_scale + pad * 2 - kernelsize) / stride + 1 last_dim = last_scale * last_scale * last_channel elif self.ff_infos[i]['ff_type'] == 'pooling': kernelsize = self.ff_infos[i]['pooling_param'].kernel_size stride = self.ff_infos[i]['pooling_param'].stride pad = self.ff_infos[i]['pooling_param'].pad print 'pool %d %d %d %d %d %d %d' % (i, kernelsize, stride, pad, last_channel, last_scale, last_dim) self.weights.append(owl.zeros([1])) self.weightsdelta.append(owl.zeros([1])) self.bias.append(owl.zeros([1])) self.biasdelta.append(owl.zeros([1])) last_channel = out_channel last_scale = (last_scale + pad * 2 - kernelsize) / stride + 1 last_dim = last_scale * last_scale * last_channel elif self.ff_infos[i]['ff_type'] == 'fully': out_channel = self.ff_infos[i]['fully_param'].num_output print 'fully %d %d %d' % (i, last_dim, out_channel) #weight if self.ff_infos[i]['fully_param'].weight_filler.type == "gaussian": self.weights.append(owl.randn([out_channel, last_dim], 0.0, self.ff_infos[i]['fully_param'].weight_filler.std)) elif self.ff_infos[i]['fully_param'].weight_filler.type == "constant": self.weights.append(owl.zeros([out_channel, last_dim]) + self.ff_infos[i]['fully_param'].weight_filler.value) else: assert False self.weightsdelta.append(owl.zeros([out_channel, last_dim])) #bias if self.ff_infos[i]['fully_param'].bias_filler.type == "gaussian": self.bias.append(owl.randn([out_channel, 1], 0.0, self.ff_infos[i]['fully_param'].weight_filler.std)) elif self.ff_infos[i]['fully_param'].bias_filler.type == "constant": self.bias.append(owl.zeros([out_channel, 1]) + self.ff_infos[i]['fully_param'].weight_filler.value) else: assert False self.biasdelta.append(owl.zeros([out_channel, 1])) last_dim = out_channel last_channel = out_channel
import owl cpu = owl.create_cpu_device() owl.set_device(cpu) x = owl.randn([1000,5000],0,64) y = x.histogram(8) print y.to_numpy()