def register_usage(self, fu_device_id, bo_device_id=None): """ Register usage of connector's forward_matrix. :param fu_device_id: context in which `forward_matrix` will be used :param bo_device_id: context in which `backward_matrix` of the connector will be calculated """ if not self.bpropagable and bo_device_id: raise ValueError( "Nobody is going to use computation from backward step. " "You mustn't register for backward propagate!") if fu_device_id != self._fo_device_id and fu_device_id not in self._f_matrices: self._f_matrices[fu_device_id] = Matrix.empty_like( self, fu_device_id) self.context[fu_device_id] = Context(fu_device_id) if bo_device_id is None: return self._f_matrices[fu_device_id] for device_id in [self._bu_device_id, bo_device_id]: if device_id not in self._b_matrices: self._b_matrices[device_id] = Matrix.empty_like( self, device_id) if device_id not in self.context: self.context[device_id] = Context(device_id) if self._bu_device_id != bo_device_id and self._bu_device_id not in self._b_matrices_pool: self._b_matrices_pool[self._bu_device_id] = Matrix.empty_like( self, self._bu_device_id) return self._f_matrices[fu_device_id], self._b_matrices[bo_device_id]
def test_bprop(self): """ compare `bprop` results for cpu and gpu backends """ r = [] for i in xrange(self.N): batch_size, dim = self.rng.random_integers(2000, size=2) y_hat = self.rng.randn(batch_size, dim).astype(dtype=np.float32) y = self.rng.randn(batch_size, dim).astype(dtype=np.float32) quagga.processor_type = 'gpu' context = Context() y_hat_gpu = Connector(Matrix.from_npa(y_hat), context, context) y_gpu = Connector(Matrix.from_npa(y)) sse_block = SseBlock(y_hat_gpu, y_gpu) sse_block.fprop() sse_block.bprop() dL_dy_hat_gpu = y_hat_gpu.backward_matrix.to_host() quagga.processor_type = 'cpu' context = Context() y_hat_cpu = Connector(Matrix.from_npa(y_hat), context, context) y_cpu = Connector(Matrix.from_npa(y)) sse_block = SseBlock(y_hat_cpu, y_cpu) sse_block.fprop() sse_block.bprop() dL_dy_hat_cpu = y_hat_cpu.backward_matrix.to_host() r.append(np.allclose(dL_dy_hat_gpu, dL_dy_hat_cpu)) self.assertEqual(sum(r), self.N)
def __init__(self, x, nonlinearity, device_id=None): """ """ self.f_context = Context(device_id) device_id = self.f_context.device_id self.learning = x.bpropagable if self.learning: self.b_context = Context(device_id) self.x, self.dL_dx = x.register_usage(device_id, device_id) self._df_dpref = Matrix.empty_like(self.x, device_id) else: self.x = x.register_usage(device_id) output = Matrix.empty_like(x, device_id) self.output = Connector(output, device_id if self.learning else None) if nonlinearity == 'sigmoid': self.f = self.x.sigmoid elif nonlinearity == 'tanh': self.f = self.x.tanh elif nonlinearity == 'relu': self.f = self.x.relu elif nonlinearity == 'softmax': raise ValueError('For softmax nonlinearity use SoftmaxBlock!') else: raise ValueError('TODO!') self.training_mode = True
def __init__(self, data, char_to_idx, batch_size, x_device_id, y_device_id): self.data = HomogeneousDataIterator(data, char_to_idx, batch_size, True, True) self.data_iterator = iter(self.data) self.x_context = Context(x_device_id) self.y_context = Context(y_device_id) max_len = 0 for sub_line in data: cur_len = len(sub_line) if cur_len > max_len: max_len = cur_len print max_len self.x = Connector( Matrix.empty(batch_size, max_len - 1, 'int', x_device_id)) self._y = Matrix.empty(batch_size, max_len - 1, 'int', y_device_id) self.y = List([Connector(self._y[:, i]) for i in xrange(max_len - 1)], self.x.ncols) self.lengths = Matrix.empty(self.x.nrows, 1, 'int', x_device_id) self._mask = Matrix.empty(self.x.nrows, self.x.ncols, 'float', x_device_id) self.mask = List( [Connector(self._mask[:, i]) for i in xrange(max_len)], self.x.ncols) self.blocking_contexts = None
def __init__(self, W, b, x, device_id=None): self.f_context = Context(device_id) device_id = self.f_context.device_id if W.bpropagable: self.W, self.dL_dW = W.register_usage(device_id, device_id) else: self.W = W.register_usage(device_id) if b: if b.bpropagable: self.b, self.dL_db = b.register_usage(device_id, device_id) self.ones = Matrix.empty(x.nrows, 1, self.b.dtype, device_id) self.ones.sync_fill(1.0) else: self.b = b.register_usage(device_id) if x.bpropagable: self.x, self.dL_dx = x.register_usage(device_id, device_id) else: self.x = x.register_usage(device_id) output = Matrix.empty(x.nrows, self.W.ncols, device_id=device_id) self.learning = hasattr(self, 'dL_dW') or hasattr(self, 'dL_db') or \ hasattr(self, 'dL_dx') if self.learning: self.b_context = Context(device_id) self.output = Connector(output, device_id) else: self.output = Connector(output)
def test_bprop(self): """ compare `bprop` results for cpu and gpu backends """ r = [] for i in xrange(self.N): max_input_sequence_len = self.rng.random_integers(500) sequence_len = max_input_sequence_len if i == 0 else self.rng.random_integers( max_input_sequence_len) batch_size = self.rng.random_integers(512) dim = self.rng.random_integers(1500) x = [ self.rng.rand(batch_size, dim).astype(dtype=np.float32) for _ in xrange(max_input_sequence_len) ] state = self.rng.get_state() quagga.processor_type = 'gpu' context = Context() x_gpu = List( [Connector(Matrix.from_npa(e), context, context) for e in x]) smean_pooling_block_gpu = SequentialMeanPoolingBlock(x_gpu) x_gpu.set_length(sequence_len) _, dL_doutput = smean_pooling_block_gpu.output.register_usage( context, context) smean_pooling_block_gpu.fprop() random_matrix = self.rng.rand(dL_doutput.nrows, dL_doutput.ncols) Matrix.from_npa(random_matrix, 'float').copy_to(context, dL_doutput) smean_pooling_block_gpu.bprop() dL_dmatrices_gpu = [e.backward_matrix.to_host() for e in x_gpu] self.rng.set_state(state) quagga.processor_type = 'cpu' context = Context() x_cpu = List( [Connector(Matrix.from_npa(e), context, context) for e in x]) smean_pooling_block_cpu = SequentialMeanPoolingBlock(x_cpu) x_cpu.set_length(sequence_len) _, dL_doutput = smean_pooling_block_cpu.output.register_usage( context, context) smean_pooling_block_cpu.fprop() random_matrix = self.rng.rand(dL_doutput.nrows, dL_doutput.ncols) Matrix.from_npa(random_matrix, 'float').copy_to(context, dL_doutput) smean_pooling_block_cpu.bprop() dL_dmatrices_cpu = [e.backward_matrix.to_host() for e in x_cpu] for dL_dmatrix_gpu, dL_dmatrix_cpu in izip(dL_dmatrices_gpu, dL_dmatrices_cpu): if not np.allclose(dL_dmatrix_gpu, dL_dmatrix_cpu): r.append(False) break else: r.append(True) self.assertEqual(sum(r), self.N)
def __init__(self, dropout_prob, x, seed=42, device_id=None): self.dropout_prob = dropout_prob self.f_context = Context(device_id) device_id = self.f_context.device_id self.generator = Matrix.get_random_generator(seed) if x.bpropagable: self.b_context = Context(device_id) self.x, self.dL_dx = x.register_usage(device_id, device_id) else: self.x = x.register_usage(device_id) self.output = Matrix.empty_like(self.x) self.output = Connector(self.output, device_id if x.bpropagable else None) self.training_mode = True
def test_theano_grad(self): quagga.processor_type = 'gpu' r = [] for i in xrange(self.N): batch_size, dim = self.rng.random_integers(2000, size=2) y_hat = self.rng.randn(batch_size, dim).astype(dtype=np.float32) y = self.rng.randn(batch_size, dim).astype(dtype=np.float32) # Theano model th_y_hat, th_y = T.fmatrix(), T.fmatrix() loss = T.mean(T.sum((th_y_hat - th_y) ** 2, axis=1)) get_theano_grads = theano.function([th_y_hat, th_y], T.grad(loss, wrt=th_y_hat)) th_dL_dy_hat = get_theano_grads(y_hat, y) # quagga model context = Context() y_hat_gpu = Connector(Matrix.from_npa(y_hat), context, context) y_gpu = Connector(Matrix.from_npa(y)) sigmoid_ce_block = SseBlock(y_hat_gpu, y_gpu) sigmoid_ce_block.fprop() sigmoid_ce_block.bprop() q_dL_dy_hat = y_hat_gpu.backward_matrix.to_host() r.append(np.allclose(th_dL_dy_hat, q_dL_dy_hat)) self.assertEqual(sum(r), self.N)
def __init__(self, x, scale_factor=1.0): self.context = Context(x.device_id) device_id = self.context.device_id self.output = x if x.bpropagable: _, self.dL_dx = x.register_usage(device_id, device_id) self.scale_factor = ct.c_float(-scale_factor)
def test_bprop_vector(self): r = [] for _ in xrange(self.N): embd_dim = self.rng.random_integers(10000) batch_size, output_dim = self.rng.random_integers(2000, size=2) W = self.get_orthogonal_matrix(embd_dim, output_dim) row_idxs = self.rng.randint(embd_dim, size=(batch_size, 1)).astype(np.int32) true_labels = self.rng.randint(output_dim, size=(batch_size, 1)).astype(np.int32) device_id = 0 output = {} for processor_type in ['gpu', 'cpu']: quagga.processor_type = processor_type qrow_idxs = Connector(Matrix.from_npa(row_idxs)) qtrue_labels = Connector(Matrix.from_npa(true_labels)) qW = Connector(Matrix.from_npa(W), device_id) row_slicing_block = RowSlicingBlock(qW, qrow_idxs) sce_block = SoftmaxCeBlock(row_slicing_block.output, qtrue_labels) qW.fprop() qrow_idxs.fprop() row_slicing_block.fprop() sce_block.fprop() sce_block.bprop() row_slicing_block.bprop() qW.add(Context(), qW.backward_matrix) output[processor_type] = qW.to_host() r.append(np.allclose(output['gpu'], output['cpu'])) self.assertEqual(sum(r), len(r))
def __init__(self, kkk, parameters, learning_rate_policy, beta1=0.9, beta2=0.999, epsilon=1e-20): self.kkk = kkk self.parameters = parameters self.m = [] self.v = [] self.contexts = [] for p in self.parameters: m = Matrix.empty_like(p) m.sync_fill(0.0) self.m.append(m) v = Matrix.empty_like(p) v.sync_fill(0.0) self.v.append(v) self.contexts.append(Context(p.device_id)) self.learning_rate_policy = learning_rate_policy self.beta1 = beta1 self.beta2 = beta2 self.epsilon = epsilon self.blocking_contexts = [] self.iteration = 0
def bprop(self): if not self.bpropagable: raise ValueError( 'Nobody was going to use computation from backward ' 'step. You should not backward propagate!') if not self._b_matrices and not self._b_sparse_matrix: # When no one registered for providing derivatives zero dense # matrix will be returned bwd = Matrix.empty_like(self, self._bu_device_id) if self._bu_device_id not in self.context: self.context[self._bu_device_id] = Context(self._bu_device_id) bwd.fill(self.context[self._bu_device_id], 0.0) self._b_matrices[self._bu_device_id] = bwd return bwd if not self._b_matrices and self._b_sparse_matrix: return self._b_sparse_matrix for bo_device_id, bwd_matrix in self._b_matrices.iteritems(): if self._bu_device_id != bo_device_id: self._b_matrices_pool[self._bu_device_id].assign( self.context[self._bu_device_id], bwd_matrix) self._b_matrices[self._bu_device_id].add( self.context[self._bu_device_id], self._b_matrices_pool[self._bu_device_id]) if self._b_sparse_matrix: self._b_matrices[self._bu_device_id].add( self.context[self._bu_device_id], self._b_sparse_matrix) return self._b_matrices[self._bu_device_id]
def __init__(self, matrix, axis=1, device_id=None): self.context = Context(device_id) self._ctype = matrix.c_dtype self._zero = self._ctype(0.0) if axis == 0: self._ones = Matrix.empty(1, matrix.nrows, matrix.dtype, device_id) self.output = Matrix.empty(1, matrix.ncols, matrix.dtype, device_id) self.alpha = self._ctype(1.0 / matrix.nrows) elif axis == 1: self._ones = Matrix.empty(matrix.ncols, 1, matrix.dtype, device_id) self.output = Matrix.empty(matrix.nrows, 1, matrix.dtype, device_id) self.alpha = None else: raise ValueError('Invalid axis!') self._ones.sync_fill(1.0) self.axis = axis if matrix.bpropagable: self.matrix, self.dL_dmatrix = matrix.register_usage( self.context, self.context) self.output = Connector(self.output, self.context, self.context) else: self.matrix = matrix.register_usage(self.context) self.output = Connector(self.output, self.context)
def __init__(self, train_data, valid_data, batch_size, word_dropout_prob, device_id): self.train_data = HomogeneousDataIterator(train_data, batch_size, randomize=True, infinite=True) self.valid_data = HomogeneousDataIterator(valid_data, batch_size) self.train_data_iterator = iter(self.train_data) self.valid_data_iterator = iter(self.valid_data) self.word_keep_prob = 1.0 - word_dropout_prob self.rnd = RandomState(47571) self.unk_idx = word_to_idx['<UNK>'] self.context = Context(device_id) c = Counter([len(line) for line in chain(train_data, valid_data)]) print c.most_common() max_len = max([len(line) for line in chain(train_data, valid_data)]) self.enc_x = Connector(Matrix.empty(batch_size, max_len, 'int', device_id)) self.enc_lengths = Matrix.empty(self.enc_x.nrows, 1, 'int', device_id) self._enc_mask = Matrix.empty(self.enc_x.nrows, self.enc_x.ncols, 'float', device_id) self.enc_mask = List([Connector(self._enc_mask[:, i]) for i in xrange(max_len)], self.enc_x.ncols) self.dec_x = Connector(Matrix.empty(batch_size, max_len + 1, 'int', device_id)) self._dec_y = Matrix.empty(batch_size, max_len + 1, 'int', device_id) self.dec_y = List([Connector(self._dec_y[:, i]) for i in xrange(max_len + 1)], self._dec_y.ncols) self.dec_lengths = Matrix.empty(self.dec_x.nrows, 1, 'int', device_id) self._dec_mask = Matrix.empty(self.dec_x.nrows, self.dec_x.ncols, 'float', device_id) self.dec_mask = List([Connector(self._dec_mask[:, i]) for i in xrange(max_len + 1)], self.dec_x.ncols) self.blocking_contexts = None self.training_mode = True
def __init__(self, x, regularization_value): self.context = Context(x.device_id) device_id = self.context.device_id if x.bpropagable: self.x, self.dL_dx = x.register_usage(device_id, device_id) else: self.x = x.register_usage(device_id) self.reg_value = ct.c_float(2 * regularization_value)
def test_bprop(self): """ compare `bprop` results for cpu and gpu backends """ r = [] for i in xrange(self.N): batch_size, x_dim = self.rng.random_integers(3000, size=2) x = self.rng.rand(batch_size, x_dim).astype(np.float32) device_id = 0 for nonlinearity in ['sigmoid', 'tanh', 'relu']: state = self.rng.get_state() quagga.processor_type = 'gpu' x_gpu = Connector(Matrix.from_npa(x), device_id) nonlinearity_block = NonlinearityBlock(x_gpu, nonlinearity) x_gpu.fprop() nonlinearity_block.fprop() _, dL_doutput = nonlinearity_block.output.register_usage( device_id, device_id) random_matrix = self.rng.rand(dL_doutput.nrows, dL_doutput.ncols) dL_doutput.assign(Context(), Matrix.from_npa(random_matrix, 'float')) nonlinearity_block.bprop() dL_dx_gpu = x_gpu.backward_matrix.to_host() self.rng.set_state(state) quagga.processor_type = 'cpu' x_cpu = Connector(Matrix.from_npa(x), device_id) nonlinearity_block = NonlinearityBlock(x_cpu, nonlinearity) x_cpu.fprop() nonlinearity_block.fprop() _, dL_doutput = nonlinearity_block.output.register_usage( device_id, device_id) random_matrix = self.rng.rand(dL_doutput.nrows, dL_doutput.ncols) dL_doutput.assign(Context(), Matrix.from_npa(random_matrix, 'float')) nonlinearity_block.bprop() dL_dx_cpu = x_cpu.backward_matrix.to_host() r.append(np.allclose(dL_dx_gpu, dL_dx_cpu)) self.assertEqual(sum(r), len(r))
def __init__(self, probs, true_labels, schedule, seed, device_id=None): self.schedule = schedule self.rnd = np.random.RandomState(seed) self.context = Context(device_id) device_id = self.context.device_id self.probs = probs.register_usage(device_id) self.true_labels = true_labels.register_usage(device_id) self.output = Connector(Matrix.empty_like(self.true_labels))
def __init__(self, x, axis, device_id=None): if axis != 1: raise NotImplementedError self.axis = axis self.context = Context(device_id) device_id = self.context.device_id self.x = x.register_usage(device_id) self.output = Connector(Matrix.empty(x.nrows, 1, x.dtype, device_id))
def __init__(self, y_hat, y, device_id=None): if y_hat.nrows != y.nrows or y_hat.ncols != y.ncols: raise ValueError('TODO!') self.context = Context(device_id) if y_hat.bpropagable: self.y_hat, self.dL_dy_hat = y_hat.register_usage(self.context, self.context) else: self.y_hat = y_hat.register_usage(self.context) self.y = y.register_usage(self.context)
def __init__(self, W, col_indexes): device_id = W.device_id self.context = Context(device_id) learning = W.bpropagable if learning: self.W, self.dL_dW = W.register_usage_with_sparse_backward_matrix() else: self.W = W.register_usage(device_id) self.col_indexes = col_indexes.register_usage(device_id) output = Matrix.empty(W.nrows, col_indexes.ncols, device_id=device_id) self.output = Connector(output, device_id if learning else None)
def __init__(self, parameters, learning_rate_policy, momentum_policy): self.parameters = parameters self.velocity = [] for p in self.parameters: v = Matrix.empty_like(p) v.sync_fill(0.0) self.velocity.append(v) self.learning_rate_policy = learning_rate_policy self.momentum_policy = momentum_policy self.contexts = [Context(p.device_id) for p in parameters] self.blocking_contexts = []
def __init__(self, x): device_id = x[0].device_id learning = x[0].bpropagable self.context = Context(device_id) self.output = Matrix.empty_like(x[0]) self.output = Connector(self.output, device_id if learning else None) if learning: self.x, self.dL_dx = izip(*x.register_usage(device_id, device_id)) else: self.x = x.register_usage(device_id) self.last_idx = x.length - 1
def __init__(self, x_sequence, y_sequence, device_id=None): """ TODO """ # TODO add during hsplit otherwise wrong accumulation of gradients if all(e.bpropagable for e in chain(x_sequence, y_sequence)): learning = True elif all(not e.bpropagable for e in chain(x_sequence, y_sequence)): learning = False else: raise ValueError('All elements should be bpropagable or ' 'non-bpropagable. Mixed state is not allowed!') x_ncols = x_sequence[0].ncols y_ncols = y_sequence[0].ncols dtype = x_sequence[0].dtype for x, y in izip(x_sequence, y_sequence): if x.ncols != x_ncols or y.ncols != y_ncols: raise ValueError( "All matrices in the sequence should have the same number of columns!" ) if x.nrows != y.nrows: raise ValueError( "Can't stack matrices in sequence with different number of rows!" ) if x.dtype != dtype or y.dtype != dtype: raise ValueError("Can't stack matrices with different dtypes!") self.context = Context(device_id) device_id = self.context.device_id if learning: self.x_sequence, self.dL_dx_sequences = izip( *x_sequence.register_usage(device_id, device_id)) self.y_sequence, self.dL_dy_sequences = izip( *y_sequence.register_usage(device_id, device_id)) self.dL_dx_sequences = List(self.dL_dx_sequences, x_sequence.length) self.dL_dy_sequences = List(self.dL_dy_sequences, y_sequence.length) else: self.x_sequence = x_sequence.register_usage(device_id) self.y_sequence = y_sequence.register_usage(device_id) self.x_sequence = List(self.x_sequence, x_sequence.length) self.y_sequence = List(self.y_sequence, y_sequence.length) output = [] for _ in xrange(x_sequence.length): matrix = Matrix.empty(x_sequence[0].nrows, x_ncols + y_ncols, dtype, device_id) output.append(Connector(matrix, device_id)) self.output = List(output, x_sequence.length) if learning: self.dL_dx_sequences = List(self.dL_dx_sequences, x_sequence.length) self.dL_dy_sequences = List(self.dL_dy_sequences, x_sequence.length)
def __init__(self, x, true_labels, mask=None, device_id=None): self.context = Context(device_id) device_id = self.context.device_id if x.bpropagable: self.x, self.dL_dx = x.register_usage(device_id, device_id) else: self.x = x.register_usage(device_id) self.true_labels = true_labels.register_usage(device_id) if mask: self.mask = mask.register_usage(device_id) self.probs = Connector(Matrix.empty_like(self.x)) self.loss = None
def __init__(self, matrices, device_id=None): self.context = Context(device_id) device_id = self.context.device_id self.output = Matrix.empty_like(matrices[0], device_id) learning = matrices[0].bpropagable self.output = Connector(self.output, device_id if learning else None) if learning: self.matrices, self.dL_dmatrices = izip( *matrices.register_usage(device_id, device_id)) else: self.matrices = matrices.register_usage(device_id) self.length = matrices.length
def __init__(self, f_matrix, bu_device_id=None): self._fo_device_id = f_matrix.device_id self._f_matrices = {self._fo_device_id: f_matrix} self.context = {self._fo_device_id: Context(self._fo_device_id)} if bu_device_id is not None: self._bu_device_id = bu_device_id self._b_matrices = dict() self._b_matrices_pool = dict() self._b_sparse_matrix = None # We need do this trick because instead we will add attribute # to the Connector instance by setting it # instead of setting attribute in f_matrix self.__f_matrix_setable_attributes = f_matrix.get_setable_attributes() for attr_name in self.__f_matrix_setable_attributes: getattr(self, attr_name)
def __init__(self, parameters, learning_rate_policy, ema_decay=0.9, epsilon=1e-6): self.parameters = parameters self.grad_sqr = [] for p in self.parameters: grad_sqr = Matrix.empty_like(p) grad_sqr.sync_fill(0.0) self.grad_sqr.append(grad_sqr) self.learning_rate_policy = learning_rate_policy self.ema_decay = ema_decay self.epsilon = epsilon self.contexts = [Context(p.device_id) for p in parameters] self.blocking_contexts = []
def __init__(self, x, repeats, axis=None, device_id=None): self.context = Context(device_id) device_id = self.context.device_id self.repeats = repeats self.axis = axis learning = x.bpropagable if learning: self.x, self.dL_dx = x.register_usage(device_id, device_id) else: self.x = x.register_usage(device_id) if axis == 0: self.output = Matrix.empty(x.nrows * repeats, x.ncols, x.dtype, device_id) elif axis == 1: self.output = Matrix.empty(x.nrows, x.ncols * repeats, x.dtype, device_id) else: raise ValueError('TODO') self.output = Connector(self.output, device_id if learning else None)
def test_theano_bprop_matrix(self): r = [] for i in xrange(self.N): max_input_sequence_len = self.rng.random_integers(300) sequence_len = max_input_sequence_len if i == 0 else self.rng.random_integers(2, max_input_sequence_len) embd_dim = self.rng.random_integers(10000) batch_size = self.rng.random_integers(500) output_dim = self.rng.random_integers(2000) W = self.get_orthogonal_matrix(embd_dim, output_dim) row_idxs = self.rng.randint(embd_dim, size=(batch_size, max_input_sequence_len)).astype(np.int32) true_labels = [self.rng.randint(output_dim, size=(batch_size, 1)).astype(np.int32) for _ in xrange(max_input_sequence_len)] device_id = 0 quagga.processor_type = 'gpu' qrow_idxs = Connector(Matrix.from_npa(row_idxs)) qtrue_labels = List([Connector(Matrix.from_npa(e)) for e in true_labels], qrow_idxs.ncols) qW = Connector(Matrix.from_npa(W), device_id) row_slicing_block = RowSlicingBlock(qW, qrow_idxs) seq_sce_block = SequencerBlock(block_class=SoftmaxCeBlock, params=[], sequences=[row_slicing_block.output, qtrue_labels]) qW.fprop() qrow_idxs.ncols = sequence_len qrow_idxs.fprop() row_slicing_block.fprop() seq_sce_block.fprop() seq_sce_block.bprop() row_slicing_block.bprop() qW.add(Context(), qW.backward_matrix) th_row_idxs = T.imatrix() th_true_labels = T.imatrix() row_slicing_layer = RowSlicingLayer(W) toutput = row_slicing_layer.get_output_expr(th_row_idxs) loss = SequentialSoftmaxLayer.get_loss(toutput, th_true_labels) dL_dW = T.grad(loss, row_slicing_layer.W) fun = theano.function([th_row_idxs, th_true_labels], updates=[(row_slicing_layer.W, row_slicing_layer.W + dL_dW)]) fun(row_idxs, np.hstack(true_labels[:sequence_len])) r.append(np.allclose(qW.to_host(), row_slicing_layer.W.get_value(), atol=1e-5)) self.assertEqual(sum(r), len(r))
def __init__(self, params, period, parameters_file_path, logger): self.params = params self.period = period self.parameters_file_path = parameters_file_path self.logger = logger self.iteration = 0 # we can use our own contexts because during Connector fprop # derivative matrices are filling with 0.0 in param's # last_usage_context, to_host changes param's last_usage_context. # We know that parameters change only during updates of optimizer. # Add derivatives can't be calculated until there are jobs to be done # in the obtaining context, which happens to be last_usage_context. # That is why we are save here. # Parameters can be changing during callbacks calls. self.context = {} for param in params.itervalues(): if param.device_id not in self.context: self.context[param.device_id] = Context(param.device_id) self.npa_params = {}