def handler(self, source_file: pathlib.Path, sim_type: str = "sim"): docs = [] indexed: int = 0 for indexed, molecule in enumerate( self.session.iterateSDFile(str(source_file)), 1 ): try: # todo add check for incremental update # todo get pubchem id? fingerprint = ( molecule.fingerprint(type=sim_type) .oneBitsList() .split(" ") ) doc = { "smiles": molecule.canonicalSmiles(), "fingerprint": fingerprint, "fingerprint_len": len(fingerprint), } docs.append({"index": {}}) docs.append(doc) if len(docs) >= 10000: info(f"Processed {indexed} rows") self.__bulk(docs) time.sleep(1) docs = [] except indigo.IndigoException as e: logging.error("Cannot upload molecule: %s", e) if len(docs) > 0: info(f"Processed {indexed} rows") self.__bulk(docs) self.es.indices.refresh(index=self.index)
def __init__(self, max_threads): if max_threads < 1: print critical('Threads number must be > 0') exit() self.max_threads = max_threads self.threads = [] print info('Start %d threads ...' % self.max_threads)
def __download(self, state: State, tries=0) -> str: source_file = pathlib.Path(state.get_state().get("source_file")) local_file = self.tmpdir / source_file.name if tries >= 20: state.change_state("download_error", "Max tries exceeded") raise TimeoutError("Max tries exceeded") with FTP( self.ftp, "anonymous", "*****@*****.**", timeout=self.timeout ) as ftp_: try: with open(local_file, "xb") as target_file: info(f"Download {source_file}") ftp_.get(source_file, target_file) except FileExistsError: local_file.unlink() self.__download(state, tries + 1) except (ftplib.error_temp, EOFError, BaseException) as err_: logging.warning("%s, sleeping 5 seconds", err_) time.sleep(5) logging.warning("retry...") self.__download(state, tries + 1) next_state = "checksum" state.change_state(next_state, "") info(f"Ok") return next_state
def test_linear_fit(self): epochs = 2000 iris = Iris() x_node = node.VarNode('x') yt_node = node.VarNode('yt') dense = node.DenseLayer(x_node, 3) softmax = act.Softmax(dense) cross_entropy = loss.CrossEntropy(softmax, yt_node) optimizer_func = core.np.Optimization.AdamOptimizer(lr=0.001) optimizer = core.np.Optimization.OptimizerIterator([x_node, yt_node], cross_entropy, optimizer_func) log_at_info() epoch = 0 ctx = node.ComputeContext() for x, y in iris.train_iterator(epochs, 8): ctx['x'], ctx['yt'] = x, y loss_now = optimizer.step(ctx, 1.0) if epoch % 100 == 0: info("[{}]\tloss_now = {}".format(epoch, loss_now)) epoch += 1 f = node.make_evaluator([x_node, yt_node], softmax) total, correct = 40, 0 for x, y_actual in iris.test_iterator(total, one_hot=False): ctx['x'], ctx['yt'] = x, y_actual y_predicted = f.at(ctx) max_idx = np.argmax(y_predicted) if max_idx == y_actual: correct += 1 percent = correct * 100 / total print("Correct= {}%".format(percent))
def test_train(self): num_iter = 100000 x_node = n.VarNode('x') y_target_node = n.VarNode('y_target') rnn_node = rnn.SimpleRnnLayer(x_node, self.name_ds.n_categories, 15) loss_node = loss.SoftmaxCrossEntropy(rnn_node, y_target_node) all_losses = [] optimizer_func = autodiff_optim.AdamOptimizer() optimizer_func = autodiff_optim.SGDOptimizer(lr=0.0001) optimizer = autodiff_optim.OptimizerIterator([x_node, y_target_node], loss_node, optimizer_func) ctx = n.ComputeContext({'x': "", 'y_target': ""}) log_at_info() every = 500 t = time.time() for i in range(1, num_iter + 1): rnn_node.set_initial_state_to_zero() c, l, category_index, name_tensor = self.name_ds.random_training_example( ) cat_tensor = self.name_ds.category_idx_to_tensor([category_index]) ctx['x'] = name_tensor ctx['y_target'] = cat_tensor ctx['i'] = i loss_value = optimizer.step(ctx, 1.0) all_losses.append(loss_value) if i % every == 0: t = time.time() - t last_10 = all_losses[-every:] av = np.average(last_10) info("[{:06d}] Avg. loss = {:10.6f}" " | {:04.2f}s per {} | Total Iters set to:{}".format( i, av, t, every, num_iter)) all_losses = [] t = time.time()
def initialize_layer(self, node: MComputeNode): class_name = node.__class__.__name__ if not (class_name in self.initializers): info("[ComputeContext.initialize_layer()] " "No initializer for: {}".format(class_name)) return initializer = self.initializers[class_name] initializer.initialize(node)
def test_linear_training_tf_fast(self): r""" For fastest results, use batch size of 64, adam optimizer and 3 epochs. You should get more than 97% accuracy :return: """ # Build the network x_node = node.VarNode('x') yt_node = node.VarNode('yt') linear1 = node.DenseLayer(x_node, 100, name="Dense-First") relu1 = act.RelUNode(linear1, name="RelU-First") linear2 = node.DenseLayer(relu1, 200, name="Dense-Second") relu2 = act.RelUNode(linear2, name="RelU-Second") linear3 = node.DenseLayer(relu2, 10, name="Dense-Third") cross_entropy = loss.LogitsCrossEntropy(linear3, yt_node, name="XEnt") # Set up optimizers and params batch_size = 64 epochs = 5 # use 25 for SGD optimizer_func = autodiff_optim.AdamOptimizer() # optimizer_func = autodiff_optim.SGDOptimizer(lr=.1) optimizer = autodiff_optim.OptimizerIterator([x_node, yt_node], cross_entropy, optimizer_func) log_at_info() losses = [] x_train, y_train, x_val, y_val, x_test, y_test = mn.load_dataset( flatten=True) iter_count = 1 predictor = node.make_evaluator([x_node, yt_node], linear3) total_time = time.time() ctx = node.ComputeContext({}) for epoch in range(epochs): epoch_time = time.time() for x, y in iterate_over_minibatches(x_train, y_train, batch_size=batch_size): ctx['x'], ctx['yt'] = x.T, to_one_hot(y, max_cat_num=9) iter_loss = optimizer.step(ctx, 1.0) / batch_size losses.append(iter_loss) iter_count += 1 epoch_time = time.time() - epoch_time loss_av = np.array(losses[:-batch_size + 1]) loss_av = np.mean(loss_av) ctx['x'], ctx['yt'] = x_val.T, to_one_hot(y_val, max_cat_num=9) y_predicted = predictor(ctx) arg_max = np.argmax(y_predicted, axis=0) correct = arg_max == y_val percent = np.mean(correct) * 100 info("Epoch {:2d}:: Validation " "accuracy:[{:5.2f}%] loss av={:01.8f}, time:{:2.3f}s".format( epoch, percent, loss_av, epoch_time)) self.assertTrue(percent > 95) total_time = time.time() - total_time info("[Mnist784DsTest.test_linear_training()] total_time = {:5.3f} s". format(total_time))
def split_test_train(self): num_data_points = len(self.targets) num_train = int(num_data_points * self.train_fraction) info("Total number of data points:{}, number of training points:{}". format(num_data_points, num_train)) self.training_set_indexes = np.random.choice(range(num_data_points), num_train, replace=False) self.test_set_indexes = set(range(num_data_points)) - set( self.training_set_indexes) self.test_set_indexes = list(self.test_set_indexes)
def test_multi_layer(self): r""" This actually performs better with SGD and normal initialization. Gets almost 99% with SGD and normal initialization :return: """ iris = Iris() x_node = node.VarNode('x') yt_node = node.VarNode('yt') dense = node.DenseLayer(x_node, 16) tanh = act.TanhNode(dense) dense2 = node.DenseLayer(tanh, 10) relu = act.RelUNode(dense2) dense3 = node.DenseLayer(relu, 3) softmax = act.Softmax(dense3) cross_entropy = loss.CrossEntropy(softmax, yt_node) #optimizer_func = core.np.Optimization.AdamOptimizer() optimizer_func = core.np.Optimization.SGDOptimizer(lr=0.01) optimizer = core.np.Optimization.OptimizerIterator([x_node, yt_node], cross_entropy, optimizer_func) log_at_info() epoch = 0 epochs = 10000 batch_size = 8 ctx = node.ComputeContext(weight_initializer=None) for x, y in iris.train_iterator(epochs, batch_size): ctx['x'], ctx['yt'] = x, y loss_now = optimizer.step(ctx, 1.0) / batch_size if epoch % 500 == 0: info("[{}]\tloss_now = {}".format(epoch, loss_now)) epoch += 1 f = node.make_evaluator([x_node, yt_node], softmax) total, correct = 100, 0 for x, y_actual in iris.test_iterator(total, one_hot=False): var_map = {'x': x, 'yt': y_actual} y_predicted = f(var_map) max_idx = np.argmax(y_predicted) mark = 'x' if max_idx == y_actual: correct += 1 mark = u'\u2713' print("X:{}, y_pred:{}, Actual={}, Predicted:{} {}".format( x.T, y_predicted.T, y_actual[0], max_idx, mark)) percent = correct * 100 / total print("Correct= {}%".format(percent)) self.assertTrue(percent > 95)
def save(self): local_cache = LocalDataCache() save_dir = local_cache.get_subdir("mnist") file_name = os.path.join(save_dir, "data784") file_name = os.path.abspath(file_name) np.save(file_name, self.data) info("Saved data array into:{}".format(file_name)) file_name = os.path.join(save_dir, "target784") file_name = os.path.abspath(file_name) np.save(file_name, self.targets) info("Saved targets array into:{}".format(file_name))
def load_data_from_cache(self): local_cache = LocalDataCache() directory = local_cache.get_subdir("mnist") file_name = os.path.join(directory, "data784.npy") file_name = os.path.abspath(file_name) info("lading data array from:{}".format(file_name)) self.data = np.load(file_name) file_name = os.path.join(directory, "target784.npy") file_name = os.path.abspath(file_name) info("Loading targets array from:{}".format(file_name)) self.targets = np.load(file_name)
def test_dropout_simple_input(self): x = np.array([[1, 2, 3], [3, 4, 1]]) x_node = node.VarNode('x') dropout = reg.Dropout(x_node) ctx = node.ComputeContext({'x': x}) x_node.forward(ctx) value = dropout.value() info( "[BatchNormalizationTest.test_dropout_simple_input()] input value = np.{}" .format(repr(x))) info( "[BatchNormalizationTest.test_dropout_simple_input()] dropout value = np.{}" .format(repr(value)))
def __bulk(self, body: Dict, tries: int = 0): if tries >= 5: raise TimeoutError try: self.es.bulk(index=self.index, body=body) except ( elasticsearch.exceptions.ConnectionTimeout, urllib3.exceptions.ReadTimeoutError, ) as err_: logging.warning(err_) time.sleep(5) info("Retrying...") self.__bulk(body, tries + 1)
def __load(self, state: State): gzipped_file = ( self.tmpdir / pathlib.Path(state.get_state().get("source_file")).name ) text_file = self.tmpdir / gzipped_file.with_suffix("").name info(f"Load {text_file}") self.elastic.handler(text_file) next_state = "clear" state.change_state(next_state, "") info(f"Ok") return next_state
def test_sigmoid(self): r""" See TestActivations.Sigmoid.ipynb for the corresponding pytorch calculations :return: """ x = np.array([[1, 2, 3, 4], [3, 4, 5, 6], [-1, 0, 1, 3]]) x_node = node.VarNode('x') target = np.zeros(x.shape) target_node = node.VarNode('target') var_map = {'x': x, 'target': target} sigmoid = SigmoidNode(x_node) l2loss = L2DistanceSquaredNorm(sigmoid, target_node) x_node.forward(var_map) target_node.forward(var_map) value = sigmoid.value() expected_value = np.array([[0.73105858, 0.88079708, 0.95257413, 0.98201379], [0.95257413, 0.98201379, 0.99330715, 0.99752738], [0.26894142, 0.5, 0.73105858, 0.95257413]]) np.testing.assert_almost_equal(expected_value, value) loss = l2loss.value() info("L2 Loss:{}".format(loss)) log_at_info() l2loss.backward(1.0, self, var_map) x_grad = x_node.total_incoming_gradient() expected_x_grad = np.array([[0.28746968, 0.18495609, 0.08606823, 0.03469004], [0.08606823, 0.03469004, 0.01320712, 0.00492082], [0.10575419, 0.25, 0.28746968, 0.08606823]]) info("-------------------------------------------------------------") info("x_grad = np.{}".format(repr(x_grad))) info("x_grad_expected= np.{}".format(repr(expected_x_grad))) np.testing.assert_almost_equal(expected_x_grad, x_grad)
def test_single_step(self): w_node = node.VarNode('w') x_node = node.VarNode('x') ya_node = node.VarNode('y_a') b_node = node.VarNode('b') w = np.array([[1, 3, 0], [0, 1, -1]]) x = (np.array([[1, -1, 2]])).T b = np.array([[-2, -3]]).T y_act = np.array([[1, 2]]).T var_map = {'w': w, 'x': x, 'y_a': y_act, 'b': b} wx_node = node.MatrixMultiplication(w_node, x_node) sum_node = node.MatrixAddition(wx_node, b_node) l2_node = L2DistanceSquaredNorm(sum_node, ya_node) w_node.forward(var_map) x_node.forward(var_map) b_node.forward(var_map) ya_node.forward(var_map) l2norm = l2_node.value() info("L2Norm: {}".format(l2norm)) y_p = w @ x + b y_del = y_p - y_act expected = np.sum(np.square(y_del)) / y_del.size debug("l2norm:{}".format(l2_node)) self.assertEqual(expected, l2norm) l2_node.backward(1.0, self, var_map) w_grad = w_node.total_incoming_gradient() b_grad = b_node.total_incoming_gradient() debug("----- w grad ------") debug(w_grad) debug("-------------------") debug("----- b grad ------") debug(b_grad) debug("-------------------") l2_node.reset_network_back() self.assertIsNone(w_node.total_incoming_gradient()) l2_node.backward(1.0, self, var_map) w_grad = w_node.total_incoming_gradient() b_grad = b_node.total_incoming_gradient() debug("----- w grad ------") debug(w_grad) debug("-------------------") debug("----- b grad ------") debug(b_grad) debug("-------------------")
def __extract(self, state: State): source_file = ( self.tmpdir / pathlib.Path(state.get_state().get("source_file")).name ) target_file = self.tmpdir / source_file.with_suffix("").name info(f"Extract {source_file}") with open(target_file, "wb") as target, gzip.open( source_file, "rb" ) as src: target.write(src.read()) next_state = "load" state.change_state(next_state, "") info(f"Ok") return next_state
def __clear(self, state: State): gzipped_file = ( self.tmpdir / pathlib.Path(state.get_state().get("source_file")).name ) text_file = self.tmpdir / gzipped_file.with_suffix("").name target_md5_file = pathlib.Path( state.get_state().get("source_file") + ".md5" ) local_md5_file = self.tmpdir / target_md5_file.name info(f"Clear") gzipped_file.unlink(missing_ok=True) text_file.unlink(missing_ok=True) local_md5_file.unlink(missing_ok=True) next_state = "complete" state.change_state(next_state, "") info(f"Ok") return next_state
def test_network_optimizer(self): w_node = node.VarNode('w', True) x_node = node.VarNode('x') ya_node = node.VarNode('y_a') b_node = node.VarNode('b', True) start_nodes = [w_node, x_node, b_node, ya_node] w = np.array([[1, 3, 0], [0, 1, -1]]) x = (np.array([[1, -1, 2]])).T b = np.array([[-2, -3]]).T y_act = np.array([[1, 2]]).T var_map = {'w': w, 'x': x, 'y_a': y_act, 'b': b} wx_node = node.MatrixMultiplication(w_node, x_node) sum_node = node.MatrixAddition(wx_node, b_node) l2_node = L2DistanceSquaredNorm(sum_node, ya_node) optimizer = optim.OptimizerIterator(start_nodes, l2_node) log_at_info() for _ in range(500): loss = optimizer.step(var_map, 1.0) info("Final loss:{}".format(loss)) self.assertTrue(math.fabs(loss) < 1e-25)
def train_iterator(self, number_times, batch_size=1, one_hot=True): r""" Will by default return one_hot encoded vectors :param number_times: :param batch_size: :param one_hot: :return: """ if batch_size >= len(self.training_set_indexes): raise Exception( "Batch size {} too large for train list of size:{}".format( batch_size, len(self.training_set_indexes))) for epoch in range(number_times): tt = time.time() row_indexes = np.random.choice(self.training_set_indexes, batch_size, replace=False) x, y = self.make_data(row_indexes, batch_size, one_hot) tt = time.time() - tt info(" [Mnist784.train_iterator()] time:{:3.6f}".format(tt)) yield epoch, x, y
def __checksum(self, state: State): source_file = pathlib.Path( state.get_state().get("source_file") + ".md5" ) local_file = self.tmpdir / source_file.name info(f"Check md5 {source_file}") with FTP( self.ftp, "anonymous", "*****@*****.**", timeout=self.timeout ) as ftp_: with open(local_file, "wb") as target_file: info(f"Download {source_file}") ftp_.get(source_file, target_file) reference_sum = "" with open(local_file, "r") as md5_ref_file: reference_sum = md5_ref_file.read().split(" ")[0] # TODO: check md5 in subprocess logging.warning("Skip md5 checksum") next_state = "extract" state.change_state(next_state, "") return next_state
def do_linear_optimization(self, optim_func, epochs=25000, batch_size=8, do_assert=True): np.random.seed(100) x_node = node.VarNode('x') y_node = node.VarNode('y') net_w = np.array([[-1, -3, 1], [0, 4, -2]]) net_b = np.array([3, -2]).reshape((2, 1)) dense = node.DenseLayer(x_node, 2, net_w, net_b) l2_node = L2DistanceSquaredNorm(dense, y_node) # optim_func = self.rate_adjustable_optimizer_func(0.01) # adam = core.np.Optimization.AdamOptimizer() optimizer = core.np.Optimization.OptimizerIterator([x_node, y_node], l2_node, optim_func) log_at_info() epoch = 0 losses = [] for x, y in self.model.data(epochs, batch_size): var_map = {'x': x, 'y': y} loss = optimizer.step(var_map, 1.0) # losses.append(loss) if epoch % 100 == 0: losses.append([epoch, loss]) if epoch % 1000 == 0: info("[{}] Loss:{}".format(epoch, loss)) epoch += 1 info("[{}] Loss:{}".format(epoch, loss)) dense_w = dense.get_w() dense_b = dense.get_b() info("w = np.{}".format(repr(dense_w))) info("b = np.{}".format(repr(dense_b))) if do_assert: np.testing.assert_array_almost_equal(dense_w, self.model_w, 3) np.testing.assert_array_almost_equal(dense_b, self.model_b, 3) return np.array(losses)
def test_dropout_with_dense(self): model_w = np.array([[1, 3, -1], [0, -4, 2.]]) model_b = np.array([-3, 2.]).reshape((2, 1)) x_node = node.VarNode('x') dense = node.DenseLayer(x_node, output_dim=2, initial_w=model_w, initial_b=model_b) p = .6 dropout = reg.Dropout(dense, dropout_prob=p) x = np.array([[1, -1], [2, 3], [-1, -2.]]) ctx = node.ComputeContext({'x': x}) found_0 = False count = 0 row_to_check = 0 while not found_0: x_node.forward(ctx) output = dropout.value() sq = np.sum(np.square(output), axis=1) found_0 = sq[row_to_check] == 0 count += 1 if count > 100: raise Exception( "Could not get 0's in first row after {} iterations.". format(count)) info("[DenseLayerStandAlone.test_single_step()] output = np.{}".format( repr(output))) dropout.backward(np.ones_like(output), self, ctx) w_grad = dense.get_w_grad() info("[DenseLayerStandAlone.test_single_step()] w_grad = np.{}".format( repr(w_grad))) b_grad = dense.get_b_grad() info("[DenseLayerStandAlone.test_single_step()] b_grad = np.{}".format( repr(b_grad))) wg_sq_sum = np.sum(np.square(w_grad), axis=1) self.assertEqual(0, wg_sq_sum[row_to_check]) bg_sum_sq = np.sum(np.square(b_grad), axis=1) self.assertEqual(0, bg_sum_sq[row_to_check]) # Test validation time (not training time) ctx.set_is_training(False) x_node.forward(ctx) test_output = dropout.value() np.testing.assert_array_almost_equal(test_output, dense.value() * p)
def test_rnn_layer_with_loss(self): debug( "[RnnLayerFullTests.test_rnn_layer_with_loss()] self.data_dir = {}" .format(self.data_dir)) x = self.name_ds.line_to_numpy('ABCD') debug("[RnnLayerFullTests.test_rnn_layer_with_loss()] ABCD: x = np.{}". format(repr(x))) debug("------------------------------------------------------") x = self.name_ds.line_to_numpy('Albert') debug( "[RnnLayerFullTests.test_rnn_layer_with_loss()] x = np.{}".format( repr(x))) debug("------------------------------------------------------") log_at_info() for i in range(5): c, l, category_index, name_tensor = self.name_ds.random_training_example( ) debug("[{}]:{}".format(c, l)) cat_tensor = self.name_ds.category_idx_to_tensor([category_index]) debug( "[RnnLayerFullTests.test_rnn_layer_with_loss()] cat_tensor = np.{}" .format(repr(cat_tensor))) x_node = n.VarNode('x') y_target_node = n.VarNode('y_target') ctx = n.ComputeContext({'x': name_tensor, 'y_target': cat_tensor}) rnn_node = rnn.SimpleRnnLayer(x_node, self.name_ds.n_categories, 128) loss_node = loss.LogitsCrossEntropy(rnn_node, y_target_node) x_node.forward(ctx) y_target_node.forward(ctx) y = rnn_node.value() info( "[RnnLayerFullTests.test_rnn_layer_with_loss()] y = np.{}".format( repr(y))) loss_value = loss_node.value() info("[RnnLayerFullTests.test_rnn_layer_with_loss()] loss = np.{}". format(repr(loss_value))) loss_node.backward(1.0, self, ctx) grads = rnn_node.total_incoming_gradient() info(grads)
def test_single_step(self): model_w = np.array([[1, 3, -1], [0, -4, 2.]]) model_b = np.array([-3, 2.]).reshape((2, 1)) x_node = node.VarNode('x') dense = node.DenseLayer(x_node, output_dim=2, initial_w=model_w, initial_b=model_b) x = np.array([[1, -1], [2, 3], [-1, -2.]]) ctx = node.ComputeContext({'x': x}) x_node.forward(ctx) output = dense.value() info("[DenseLayerStandAlone.test_single_step()] output = np.{}".format( repr(output))) dense.backward(np.ones_like(output), self, ctx) w_grad = dense.get_w_grad() info("[DenseLayerStandAlone.test_single_step()] w_grad = np.{}".format( repr(w_grad))) b_grad = dense.get_b_grad() info("[DenseLayerStandAlone.test_single_step()] b_grad = np.{}".format( repr(b_grad)))
def test_convolution_with_l2(self): img = np.array([[1, 2, 3, 4], [3, 4, 5, 6], [-1, 0, 1, 3], [0, 2, -1, 4]]) kernel = np.array([[1, -1], [0, 2]]) y = np.ones((3, 3)) img_node = node.VarNode('img') c2d = conv.Convolution2D(img_node, input_shape=(4, 4), kernel=kernel) target_node = node.VarNode('y') l2 = L2DistanceSquaredNorm(c2d, target_node) var_map = {'img': img, 'y': y} img_node.forward(var_map) target_node.forward(var_map) info("Original x into the convolution layer") info(repr(img)) output_image = c2d.value() info("Output of the convolution layer") expected_output = np.array([[7., 9., 11.], [-1., 1., 5.], [3., -3., 6.]]) np.testing.assert_array_almost_equal(expected_output, output_image) info(repr(output_image)) log_at_info() info("Kernel before gradient descent") info(repr(c2d.get_kernel())) optim_func = self.rate_adjustable_optimizer_func(0.001) optimizer = core.np.Optimization.OptimizerIterator([img_node, target_node], l2, optim_func) loss = optimizer.step(var_map, 1.0) info("Took a single gradient descent step - calculated weights and updated gradients") info("<<<<Printing loss matrix after single step>>>>") info(repr(loss)) info("Printing kernel:") info(repr(c2d.get_kernel())) info("--------------------------------------") info("Printing kernel gradient:") info(repr(c2d.get_kernel_grad())) info("-------------------------") info("Bias :{}".format(c2d.get_bias())) info("Bias gradient :{}".format(c2d.get_bias_grad())) expected_kernel = np.array([[0.98466667, -1.02288889], [-0.02066667, 1.96355556]]) np.testing.assert_array_almost_equal(expected_kernel, c2d.get_kernel()) expected_kernel_grad = np.array([[15.33333333, 22.88888889], [20.66666667, 36.44444444]]) np.testing.assert_array_almost_equal(expected_kernel_grad, c2d.get_kernel_grad()) expected_bias = -0.0064444444444444445 expected_bias_grad = 6.444444444444445 np.testing.assert_almost_equal(expected_bias, c2d.get_bias()) np.testing.assert_almost_equal(expected_bias_grad, c2d.get_bias_grad())
def test_convolution_small(self): img = np.array([[1, 2, 3, 4], [3, 4, 5, 6], [-1, 0, 1, 3], [0, 2, -1, 4]]) kernel = np.array([[1, -1], [0, 2]]) img_node = node.VarNode('img') c2d = conv.Convolution2D(img_node, input_shape=(4, 4), kernel=kernel) var_map = {'img': img} img_node.forward(var_map) info("Original x into the convolution layer") info(repr(img)) output_image = c2d.value() info("Output of the convolution layer") expected_output = np.array([[7., 9., 11.], [-1., 1., 5.], [3., -3., 6.]]) np.testing.assert_array_almost_equal(expected_output, output_image) info(repr(output_image)) log_at_info() c2d.backward(output_image * 0.1, self, var_map) info("Kernel before gradient descent") info(repr(c2d.get_kernel())) def optimizer_function(_w, grad, local_node_storage={}): return _w - 0.001 * grad optimizer = core.np.Optimization.OptimizerIterator([img_node], c2d, optimizer_function) loss = optimizer.step(var_map, np.ones_like(output_image)) info("Printing loss matrix - not really loss but just the output of the last node") info(repr(loss)) info("Printing kernel after gradient descent") info(repr(c2d.get_kernel())) expected_kernel = np.array([[0.998, -1.003111], [-1.444444444e-3, 1.9973333]]) info("kernel gradient:{}".format(repr(c2d.kernel_grad))) np.testing.assert_array_almost_equal(expected_kernel, c2d.get_kernel()) self.assertAlmostEqual(-0.001, c2d.get_bias()) info("Bias after gradient descent:{}".format(c2d.get_bias())) info("Gradient of bias :{}".format(c2d.bias_grad))
def test_linear_training(self): r""" For fastest results, use batch size of 64, adam optimizer and 3 epochs. You should get more than 97% accuracy :return: """ # Build the network x_node = node.VarNode('x') yt_node = node.VarNode('yt') linear1 = node.DenseLayer(x_node, 100, name="Dense-First", weight_scale=0.01) relu1 = act.RelUNode(linear1, name="RelU-First") linear2 = node.DenseLayer(relu1, 200, name="Dense-Second", weight_scale=0.01) relu2 = act.RelUNode(linear2, name="RelU-Second") linear3 = node.DenseLayer(relu2, 10, name="Dense-Third", weight_scale=0.01) cross_entropy = loss.LogitsCrossEntropy(linear3, yt_node, name="XEnt") # Set up optimizers and params batch_size = 64 epochs = 3 optimizer_func = autodiff_optim.AdamOptimizer() # optimizer_func = autodiff_optim.SGDOptimizer(lr=.1) optimizer = autodiff_optim.OptimizerIterator([x_node, yt_node], cross_entropy, optimizer_func) log_at_info() losses = [] iter_count = 1 predictor = node.make_evaluator([x_node, yt_node], linear3) ctx = node.ComputeContext({}) mnist = Mnist784() total_time = time.time() for epoch in range(epochs): epoch_time = time.time() iter = 0 for x, y in mnist.train_iterator_seq(batch_size=batch_size): ctx['x'], ctx['yt'] = x, y iter_loss = optimizer.step(ctx, 1.0) / batch_size losses.append(iter_loss) iter += 1 if iter % 100 == 0: print("iter:{}".format(iter)) loss_av = np.array(losses[:-batch_size + 1]) loss_av = np.mean(loss_av) e, xv, yv = mnist.test_iterator(1, batch_size=-1, one_hot=False) ctx['x'], ctx['yt'] = xv, yv percent = self.measure_validation_perf(predictor, ctx, yv) epoch_time = time.time() - epoch_time info("Iter {:2d}:: Val:{:2.4f}% , loss av={:01.8f}, time:{:2.3f}s". format(epoch, percent, loss_av, epoch_time)) total_time = time.time() - total_time info("Total time taken:{:4.4f}".format(total_time))
# syshelp elif command == 'help -sys': core.assist_sys() # geom help elif command == 'help -geom': core.assist_geom() # math elif command == 'math': core.climath() # apache elif command == 'info': core.info() # right triangle checker elif command == 'rightcheck': core.rightcheck() # quadratic formula elif command == 'quadratic': core.quadratic() # find parts of right triangle elif command == 'pythagorean': core.pythagorean() # find midpoint elif command == 'midpoint':
def test_forward(self): self.forward() value = self.max_pool_node.value() info("maxpool = np.{}".format(repr(value))) expected_value = np.array([[2, 3, 4], [9, 9, 5]]) np.testing.assert_almost_equal(expected_value, value)