def handler(self, source_file: pathlib.Path, sim_type: str = "sim"):
docs = []
indexed: int = 0
for indexed, molecule in enumerate(
self.session.iterateSDFile(str(source_file)), 1
):
try:
# todo add check for incremental update
# todo get pubchem id?
fingerprint = (
molecule.fingerprint(type=sim_type)
.oneBitsList()
.split(" ")
)
doc = {
"smiles": molecule.canonicalSmiles(),
"fingerprint": fingerprint,
"fingerprint_len": len(fingerprint),
}
docs.append({"index": {}})
docs.append(doc)
if len(docs) >= 10000:
info(f"Processed {indexed} rows")
self.__bulk(docs)
time.sleep(1)
docs = []
except indigo.IndigoException as e:
logging.error("Cannot upload molecule: %s", e)
if len(docs) > 0:
info(f"Processed {indexed} rows")
self.__bulk(docs)
self.es.indices.refresh(index=self.index)
def __init__(self, max_threads):
if max_threads < 1:
print critical('Threads number must be > 0')
exit()
self.max_threads = max_threads
self.threads = []
print info('Start %d threads ...' % self.max_threads)
def __download(self, state: State, tries=0) -> str:
source_file = pathlib.Path(state.get_state().get("source_file"))
local_file = self.tmpdir / source_file.name
if tries >= 20:
state.change_state("download_error", "Max tries exceeded")
raise TimeoutError("Max tries exceeded")
with FTP(
self.ftp, "anonymous", "*****@*****.**", timeout=self.timeout
) as ftp_:
try:
with open(local_file, "xb") as target_file:
info(f"Download {source_file}")
ftp_.get(source_file, target_file)
except FileExistsError:
local_file.unlink()
self.__download(state, tries + 1)
except (ftplib.error_temp, EOFError, BaseException) as err_:
logging.warning("%s, sleeping 5 seconds", err_)
time.sleep(5)
logging.warning("retry...")
self.__download(state, tries + 1)
next_state = "checksum"
state.change_state(next_state, "")
info(f"Ok")
return next_state
def test_linear_fit(self):
epochs = 2000
iris = Iris()
x_node = node.VarNode('x')
yt_node = node.VarNode('yt')
dense = node.DenseLayer(x_node, 3)
softmax = act.Softmax(dense)
cross_entropy = loss.CrossEntropy(softmax, yt_node)
optimizer_func = core.np.Optimization.AdamOptimizer(lr=0.001)
optimizer = core.np.Optimization.OptimizerIterator([x_node, yt_node],
cross_entropy,
optimizer_func)
log_at_info()
epoch = 0
ctx = node.ComputeContext()
for x, y in iris.train_iterator(epochs, 8):
ctx['x'], ctx['yt'] = x, y
loss_now = optimizer.step(ctx, 1.0)
if epoch % 100 == 0:
info("[{}]\tloss_now = {}".format(epoch, loss_now))
epoch += 1
f = node.make_evaluator([x_node, yt_node], softmax)
total, correct = 40, 0
for x, y_actual in iris.test_iterator(total, one_hot=False):
ctx['x'], ctx['yt'] = x, y_actual
y_predicted = f.at(ctx)
max_idx = np.argmax(y_predicted)
if max_idx == y_actual:
correct += 1
percent = correct * 100 / total
print("Correct= {}%".format(percent))
def test_train(self):
num_iter = 100000
x_node = n.VarNode('x')
y_target_node = n.VarNode('y_target')
rnn_node = rnn.SimpleRnnLayer(x_node, self.name_ds.n_categories, 15)
loss_node = loss.SoftmaxCrossEntropy(rnn_node, y_target_node)
all_losses = []
optimizer_func = autodiff_optim.AdamOptimizer()
optimizer_func = autodiff_optim.SGDOptimizer(lr=0.0001)
optimizer = autodiff_optim.OptimizerIterator([x_node, y_target_node],
loss_node, optimizer_func)
ctx = n.ComputeContext({'x': "", 'y_target': ""})
log_at_info()
every = 500
t = time.time()
for i in range(1, num_iter + 1):
rnn_node.set_initial_state_to_zero()
c, l, category_index, name_tensor = self.name_ds.random_training_example(
)
cat_tensor = self.name_ds.category_idx_to_tensor([category_index])
ctx['x'] = name_tensor
ctx['y_target'] = cat_tensor
ctx['i'] = i
loss_value = optimizer.step(ctx, 1.0)
all_losses.append(loss_value)
if i % every == 0:
t = time.time() - t
last_10 = all_losses[-every:]
av = np.average(last_10)
info("[{:06d}] Avg. loss = {:10.6f}"
" | {:04.2f}s per {} | Total Iters set to:{}".format(
i, av, t, every, num_iter))
all_losses = []
t = time.time()
def initialize_layer(self, node: MComputeNode):
class_name = node.__class__.__name__
if not (class_name in self.initializers):
info("[ComputeContext.initialize_layer()] "
"No initializer for: {}".format(class_name))
return
initializer = self.initializers[class_name]
initializer.initialize(node)
def test_linear_training_tf_fast(self):
r"""
For fastest results, use batch size of 64, adam optimizer
and 3 epochs. You should get more than 97% accuracy
:return:
"""
# Build the network
x_node = node.VarNode('x')
yt_node = node.VarNode('yt')
linear1 = node.DenseLayer(x_node, 100, name="Dense-First")
relu1 = act.RelUNode(linear1, name="RelU-First")
linear2 = node.DenseLayer(relu1, 200, name="Dense-Second")
relu2 = act.RelUNode(linear2, name="RelU-Second")
linear3 = node.DenseLayer(relu2, 10, name="Dense-Third")
cross_entropy = loss.LogitsCrossEntropy(linear3, yt_node, name="XEnt")
# Set up optimizers and params
batch_size = 64
epochs = 5 # use 25 for SGD
optimizer_func = autodiff_optim.AdamOptimizer()
# optimizer_func = autodiff_optim.SGDOptimizer(lr=.1)
optimizer = autodiff_optim.OptimizerIterator([x_node, yt_node],
cross_entropy,
optimizer_func)
log_at_info()
losses = []
x_train, y_train, x_val, y_val, x_test, y_test = mn.load_dataset(
flatten=True)
iter_count = 1
predictor = node.make_evaluator([x_node, yt_node], linear3)
total_time = time.time()
ctx = node.ComputeContext({})
for epoch in range(epochs):
epoch_time = time.time()
for x, y in iterate_over_minibatches(x_train,
y_train,
batch_size=batch_size):
ctx['x'], ctx['yt'] = x.T, to_one_hot(y, max_cat_num=9)
iter_loss = optimizer.step(ctx, 1.0) / batch_size
losses.append(iter_loss)
iter_count += 1
epoch_time = time.time() - epoch_time
loss_av = np.array(losses[:-batch_size + 1])
loss_av = np.mean(loss_av)
ctx['x'], ctx['yt'] = x_val.T, to_one_hot(y_val, max_cat_num=9)
y_predicted = predictor(ctx)
arg_max = np.argmax(y_predicted, axis=0)
correct = arg_max == y_val
percent = np.mean(correct) * 100
info("Epoch {:2d}:: Validation "
"accuracy:[{:5.2f}%] loss av={:01.8f}, time:{:2.3f}s".format(
epoch, percent, loss_av, epoch_time))
self.assertTrue(percent > 95)
total_time = time.time() - total_time
info("[Mnist784DsTest.test_linear_training()] total_time = {:5.3f} s".
format(total_time))
def split_test_train(self):
num_data_points = len(self.targets)
num_train = int(num_data_points * self.train_fraction)
info("Total number of data points:{}, number of training points:{}".
format(num_data_points, num_train))
self.training_set_indexes = np.random.choice(range(num_data_points),
num_train,
replace=False)
self.test_set_indexes = set(range(num_data_points)) - set(
self.training_set_indexes)
self.test_set_indexes = list(self.test_set_indexes)
def test_multi_layer(self):
r"""
This actually performs better with SGD and normal initialization.
Gets almost 99% with SGD and normal initialization
:return:
"""
iris = Iris()
x_node = node.VarNode('x')
yt_node = node.VarNode('yt')
dense = node.DenseLayer(x_node, 16)
tanh = act.TanhNode(dense)
dense2 = node.DenseLayer(tanh, 10)
relu = act.RelUNode(dense2)
dense3 = node.DenseLayer(relu, 3)
softmax = act.Softmax(dense3)
cross_entropy = loss.CrossEntropy(softmax, yt_node)
#optimizer_func = core.np.Optimization.AdamOptimizer()
optimizer_func = core.np.Optimization.SGDOptimizer(lr=0.01)
optimizer = core.np.Optimization.OptimizerIterator([x_node, yt_node],
cross_entropy,
optimizer_func)
log_at_info()
epoch = 0
epochs = 10000
batch_size = 8
ctx = node.ComputeContext(weight_initializer=None)
for x, y in iris.train_iterator(epochs, batch_size):
ctx['x'], ctx['yt'] = x, y
loss_now = optimizer.step(ctx, 1.0) / batch_size
if epoch % 500 == 0:
info("[{}]\tloss_now = {}".format(epoch, loss_now))
epoch += 1
f = node.make_evaluator([x_node, yt_node], softmax)
total, correct = 100, 0
for x, y_actual in iris.test_iterator(total, one_hot=False):
var_map = {'x': x, 'yt': y_actual}
y_predicted = f(var_map)
max_idx = np.argmax(y_predicted)
mark = 'x'
if max_idx == y_actual:
correct += 1
mark = u'\u2713'
print("X:{}, y_pred:{}, Actual={}, Predicted:{} {}".format(
x.T, y_predicted.T, y_actual[0], max_idx, mark))
percent = correct * 100 / total
print("Correct= {}%".format(percent))
self.assertTrue(percent > 95)
def save(self):
local_cache = LocalDataCache()
save_dir = local_cache.get_subdir("mnist")
file_name = os.path.join(save_dir, "data784")
file_name = os.path.abspath(file_name)
np.save(file_name, self.data)
info("Saved data array into:{}".format(file_name))
file_name = os.path.join(save_dir, "target784")
file_name = os.path.abspath(file_name)
np.save(file_name, self.targets)
info("Saved targets array into:{}".format(file_name))
def load_data_from_cache(self):
local_cache = LocalDataCache()
directory = local_cache.get_subdir("mnist")
file_name = os.path.join(directory, "data784.npy")
file_name = os.path.abspath(file_name)
info("lading data array from:{}".format(file_name))
self.data = np.load(file_name)
file_name = os.path.join(directory, "target784.npy")
file_name = os.path.abspath(file_name)
info("Loading targets array from:{}".format(file_name))
self.targets = np.load(file_name)
def test_dropout_simple_input(self):
x = np.array([[1, 2, 3], [3, 4, 1]])
x_node = node.VarNode('x')
dropout = reg.Dropout(x_node)
ctx = node.ComputeContext({'x': x})
x_node.forward(ctx)
value = dropout.value()
info(
"[BatchNormalizationTest.test_dropout_simple_input()] input value = np.{}"
.format(repr(x)))
info(
"[BatchNormalizationTest.test_dropout_simple_input()] dropout value = np.{}"
.format(repr(value)))
def __bulk(self, body: Dict, tries: int = 0):
if tries >= 5:
raise TimeoutError
try:
self.es.bulk(index=self.index, body=body)
except (
elasticsearch.exceptions.ConnectionTimeout,
urllib3.exceptions.ReadTimeoutError,
) as err_:
logging.warning(err_)
time.sleep(5)
info("Retrying...")
self.__bulk(body, tries + 1)
def __load(self, state: State):
gzipped_file = (
self.tmpdir
/ pathlib.Path(state.get_state().get("source_file")).name
)
text_file = self.tmpdir / gzipped_file.with_suffix("").name
info(f"Load {text_file}")
self.elastic.handler(text_file)
next_state = "clear"
state.change_state(next_state, "")
info(f"Ok")
return next_state
def test_sigmoid(self):
r"""
See TestActivations.Sigmoid.ipynb for the corresponding pytorch calculations
:return:
"""
x = np.array([[1, 2, 3, 4], [3, 4, 5, 6], [-1, 0, 1, 3]])
x_node = node.VarNode('x')
target = np.zeros(x.shape)
target_node = node.VarNode('target')
var_map = {'x': x, 'target': target}
sigmoid = SigmoidNode(x_node)
l2loss = L2DistanceSquaredNorm(sigmoid, target_node)
x_node.forward(var_map)
target_node.forward(var_map)
value = sigmoid.value()
expected_value = np.array([[0.73105858, 0.88079708, 0.95257413, 0.98201379],
[0.95257413, 0.98201379, 0.99330715, 0.99752738],
[0.26894142, 0.5, 0.73105858, 0.95257413]])
np.testing.assert_almost_equal(expected_value, value)
loss = l2loss.value()
info("L2 Loss:{}".format(loss))
log_at_info()
l2loss.backward(1.0, self, var_map)
x_grad = x_node.total_incoming_gradient()
expected_x_grad = np.array([[0.28746968, 0.18495609, 0.08606823, 0.03469004],
[0.08606823, 0.03469004, 0.01320712, 0.00492082],
[0.10575419, 0.25, 0.28746968, 0.08606823]])
info("-------------------------------------------------------------")
info("x_grad = np.{}".format(repr(x_grad)))
info("x_grad_expected= np.{}".format(repr(expected_x_grad)))
np.testing.assert_almost_equal(expected_x_grad, x_grad)
def test_single_step(self):
w_node = node.VarNode('w')
x_node = node.VarNode('x')
ya_node = node.VarNode('y_a')
b_node = node.VarNode('b')
w = np.array([[1, 3, 0], [0, 1, -1]])
x = (np.array([[1, -1, 2]])).T
b = np.array([[-2, -3]]).T
y_act = np.array([[1, 2]]).T
var_map = {'w': w, 'x': x, 'y_a': y_act, 'b': b}
wx_node = node.MatrixMultiplication(w_node, x_node)
sum_node = node.MatrixAddition(wx_node, b_node)
l2_node = L2DistanceSquaredNorm(sum_node, ya_node)
w_node.forward(var_map)
x_node.forward(var_map)
b_node.forward(var_map)
ya_node.forward(var_map)
l2norm = l2_node.value()
info("L2Norm: {}".format(l2norm))
y_p = w @ x + b
y_del = y_p - y_act
expected = np.sum(np.square(y_del)) / y_del.size
debug("l2norm:{}".format(l2_node))
self.assertEqual(expected, l2norm)
l2_node.backward(1.0, self, var_map)
w_grad = w_node.total_incoming_gradient()
b_grad = b_node.total_incoming_gradient()
debug("----- w grad ------")
debug(w_grad)
debug("-------------------")
debug("----- b grad ------")
debug(b_grad)
debug("-------------------")
l2_node.reset_network_back()
self.assertIsNone(w_node.total_incoming_gradient())
l2_node.backward(1.0, self, var_map)
w_grad = w_node.total_incoming_gradient()
b_grad = b_node.total_incoming_gradient()
debug("----- w grad ------")
debug(w_grad)
debug("-------------------")
debug("----- b grad ------")
debug(b_grad)
debug("-------------------")
def __extract(self, state: State):
source_file = (
self.tmpdir
/ pathlib.Path(state.get_state().get("source_file")).name
)
target_file = self.tmpdir / source_file.with_suffix("").name
info(f"Extract {source_file}")
with open(target_file, "wb") as target, gzip.open(
source_file, "rb"
) as src:
target.write(src.read())
next_state = "load"
state.change_state(next_state, "")
info(f"Ok")
return next_state
def __clear(self, state: State):
gzipped_file = (
self.tmpdir
/ pathlib.Path(state.get_state().get("source_file")).name
)
text_file = self.tmpdir / gzipped_file.with_suffix("").name
target_md5_file = pathlib.Path(
state.get_state().get("source_file") + ".md5"
)
local_md5_file = self.tmpdir / target_md5_file.name
info(f"Clear")
gzipped_file.unlink(missing_ok=True)
text_file.unlink(missing_ok=True)
local_md5_file.unlink(missing_ok=True)
next_state = "complete"
state.change_state(next_state, "")
info(f"Ok")
return next_state
def test_network_optimizer(self):
w_node = node.VarNode('w', True)
x_node = node.VarNode('x')
ya_node = node.VarNode('y_a')
b_node = node.VarNode('b', True)
start_nodes = [w_node, x_node, b_node, ya_node]
w = np.array([[1, 3, 0], [0, 1, -1]])
x = (np.array([[1, -1, 2]])).T
b = np.array([[-2, -3]]).T
y_act = np.array([[1, 2]]).T
var_map = {'w': w, 'x': x, 'y_a': y_act, 'b': b}
wx_node = node.MatrixMultiplication(w_node, x_node)
sum_node = node.MatrixAddition(wx_node, b_node)
l2_node = L2DistanceSquaredNorm(sum_node, ya_node)
optimizer = optim.OptimizerIterator(start_nodes, l2_node)
log_at_info()
for _ in range(500):
loss = optimizer.step(var_map, 1.0)
info("Final loss:{}".format(loss))
self.assertTrue(math.fabs(loss) < 1e-25)
def train_iterator(self, number_times, batch_size=1, one_hot=True):
r"""
Will by default return one_hot encoded vectors
:param number_times:
:param batch_size:
:param one_hot:
:return:
"""
if batch_size >= len(self.training_set_indexes):
raise Exception(
"Batch size {} too large for train list of size:{}".format(
batch_size, len(self.training_set_indexes)))
for epoch in range(number_times):
tt = time.time()
row_indexes = np.random.choice(self.training_set_indexes,
batch_size,
replace=False)
x, y = self.make_data(row_indexes, batch_size, one_hot)
tt = time.time() - tt
info(" [Mnist784.train_iterator()] time:{:3.6f}".format(tt))
yield epoch, x, y
def __checksum(self, state: State):
source_file = pathlib.Path(
state.get_state().get("source_file") + ".md5"
)
local_file = self.tmpdir / source_file.name
info(f"Check md5 {source_file}")
with FTP(
self.ftp, "anonymous", "*****@*****.**", timeout=self.timeout
) as ftp_:
with open(local_file, "wb") as target_file:
info(f"Download {source_file}")
ftp_.get(source_file, target_file)
reference_sum = ""
with open(local_file, "r") as md5_ref_file:
reference_sum = md5_ref_file.read().split(" ")[0]
# TODO: check md5 in subprocess
logging.warning("Skip md5 checksum")
next_state = "extract"
state.change_state(next_state, "")
return next_state
def do_linear_optimization(self,
optim_func,
epochs=25000,
batch_size=8,
do_assert=True):
np.random.seed(100)
x_node = node.VarNode('x')
y_node = node.VarNode('y')
net_w = np.array([[-1, -3, 1], [0, 4, -2]])
net_b = np.array([3, -2]).reshape((2, 1))
dense = node.DenseLayer(x_node, 2, net_w, net_b)
l2_node = L2DistanceSquaredNorm(dense, y_node)
# optim_func = self.rate_adjustable_optimizer_func(0.01)
# adam = core.np.Optimization.AdamOptimizer()
optimizer = core.np.Optimization.OptimizerIterator([x_node, y_node],
l2_node, optim_func)
log_at_info()
epoch = 0
losses = []
for x, y in self.model.data(epochs, batch_size):
var_map = {'x': x, 'y': y}
loss = optimizer.step(var_map, 1.0)
# losses.append(loss)
if epoch % 100 == 0:
losses.append([epoch, loss])
if epoch % 1000 == 0:
info("[{}] Loss:{}".format(epoch, loss))
epoch += 1
info("[{}] Loss:{}".format(epoch, loss))
dense_w = dense.get_w()
dense_b = dense.get_b()
info("w = np.{}".format(repr(dense_w)))
info("b = np.{}".format(repr(dense_b)))
if do_assert:
np.testing.assert_array_almost_equal(dense_w, self.model_w, 3)
np.testing.assert_array_almost_equal(dense_b, self.model_b, 3)
return np.array(losses)
def test_dropout_with_dense(self):
model_w = np.array([[1, 3, -1], [0, -4, 2.]])
model_b = np.array([-3, 2.]).reshape((2, 1))
x_node = node.VarNode('x')
dense = node.DenseLayer(x_node,
output_dim=2,
initial_w=model_w,
initial_b=model_b)
p = .6
dropout = reg.Dropout(dense, dropout_prob=p)
x = np.array([[1, -1], [2, 3], [-1, -2.]])
ctx = node.ComputeContext({'x': x})
found_0 = False
count = 0
row_to_check = 0
while not found_0:
x_node.forward(ctx)
output = dropout.value()
sq = np.sum(np.square(output), axis=1)
found_0 = sq[row_to_check] == 0
count += 1
if count > 100:
raise Exception(
"Could not get 0's in first row after {} iterations.".
format(count))
info("[DenseLayerStandAlone.test_single_step()] output = np.{}".format(
repr(output)))
dropout.backward(np.ones_like(output), self, ctx)
w_grad = dense.get_w_grad()
info("[DenseLayerStandAlone.test_single_step()] w_grad = np.{}".format(
repr(w_grad)))
b_grad = dense.get_b_grad()
info("[DenseLayerStandAlone.test_single_step()] b_grad = np.{}".format(
repr(b_grad)))
wg_sq_sum = np.sum(np.square(w_grad), axis=1)
self.assertEqual(0, wg_sq_sum[row_to_check])
bg_sum_sq = np.sum(np.square(b_grad), axis=1)
self.assertEqual(0, bg_sum_sq[row_to_check])
# Test validation time (not training time)
ctx.set_is_training(False)
x_node.forward(ctx)
test_output = dropout.value()
np.testing.assert_array_almost_equal(test_output, dense.value() * p)
def test_rnn_layer_with_loss(self):
debug(
"[RnnLayerFullTests.test_rnn_layer_with_loss()] self.data_dir = {}"
.format(self.data_dir))
x = self.name_ds.line_to_numpy('ABCD')
debug("[RnnLayerFullTests.test_rnn_layer_with_loss()] ABCD: x = np.{}".
format(repr(x)))
debug("------------------------------------------------------")
x = self.name_ds.line_to_numpy('Albert')
debug(
"[RnnLayerFullTests.test_rnn_layer_with_loss()] x = np.{}".format(
repr(x)))
debug("------------------------------------------------------")
log_at_info()
for i in range(5):
c, l, category_index, name_tensor = self.name_ds.random_training_example(
)
debug("[{}]:{}".format(c, l))
cat_tensor = self.name_ds.category_idx_to_tensor([category_index])
debug(
"[RnnLayerFullTests.test_rnn_layer_with_loss()] cat_tensor = np.{}"
.format(repr(cat_tensor)))
x_node = n.VarNode('x')
y_target_node = n.VarNode('y_target')
ctx = n.ComputeContext({'x': name_tensor, 'y_target': cat_tensor})
rnn_node = rnn.SimpleRnnLayer(x_node, self.name_ds.n_categories, 128)
loss_node = loss.LogitsCrossEntropy(rnn_node, y_target_node)
x_node.forward(ctx)
y_target_node.forward(ctx)
y = rnn_node.value()
info(
"[RnnLayerFullTests.test_rnn_layer_with_loss()] y = np.{}".format(
repr(y)))
loss_value = loss_node.value()
info("[RnnLayerFullTests.test_rnn_layer_with_loss()] loss = np.{}".
format(repr(loss_value)))
loss_node.backward(1.0, self, ctx)
grads = rnn_node.total_incoming_gradient()
info(grads)
def test_single_step(self):
model_w = np.array([[1, 3, -1], [0, -4, 2.]])
model_b = np.array([-3, 2.]).reshape((2, 1))
x_node = node.VarNode('x')
dense = node.DenseLayer(x_node,
output_dim=2,
initial_w=model_w,
initial_b=model_b)
x = np.array([[1, -1], [2, 3], [-1, -2.]])
ctx = node.ComputeContext({'x': x})
x_node.forward(ctx)
output = dense.value()
info("[DenseLayerStandAlone.test_single_step()] output = np.{}".format(
repr(output)))
dense.backward(np.ones_like(output), self, ctx)
w_grad = dense.get_w_grad()
info("[DenseLayerStandAlone.test_single_step()] w_grad = np.{}".format(
repr(w_grad)))
b_grad = dense.get_b_grad()
info("[DenseLayerStandAlone.test_single_step()] b_grad = np.{}".format(
repr(b_grad)))
def test_convolution_with_l2(self):
img = np.array([[1, 2, 3, 4], [3, 4, 5, 6], [-1, 0, 1, 3], [0, 2, -1, 4]])
kernel = np.array([[1, -1], [0, 2]])
y = np.ones((3, 3))
img_node = node.VarNode('img')
c2d = conv.Convolution2D(img_node, input_shape=(4, 4), kernel=kernel)
target_node = node.VarNode('y')
l2 = L2DistanceSquaredNorm(c2d, target_node)
var_map = {'img': img, 'y': y}
img_node.forward(var_map)
target_node.forward(var_map)
info("Original x into the convolution layer")
info(repr(img))
output_image = c2d.value()
info("Output of the convolution layer")
expected_output = np.array([[7., 9., 11.],
[-1., 1., 5.],
[3., -3., 6.]])
np.testing.assert_array_almost_equal(expected_output, output_image)
info(repr(output_image))
log_at_info()
info("Kernel before gradient descent")
info(repr(c2d.get_kernel()))
optim_func = self.rate_adjustable_optimizer_func(0.001)
optimizer = core.np.Optimization.OptimizerIterator([img_node, target_node], l2, optim_func)
loss = optimizer.step(var_map, 1.0)
info("Took a single gradient descent step - calculated weights and updated gradients")
info("<<<<Printing loss matrix after single step>>>>")
info(repr(loss))
info("Printing kernel:")
info(repr(c2d.get_kernel()))
info("--------------------------------------")
info("Printing kernel gradient:")
info(repr(c2d.get_kernel_grad()))
info("-------------------------")
info("Bias :{}".format(c2d.get_bias()))
info("Bias gradient :{}".format(c2d.get_bias_grad()))
expected_kernel = np.array([[0.98466667, -1.02288889],
[-0.02066667, 1.96355556]])
np.testing.assert_array_almost_equal(expected_kernel, c2d.get_kernel())
expected_kernel_grad = np.array([[15.33333333, 22.88888889],
[20.66666667, 36.44444444]])
np.testing.assert_array_almost_equal(expected_kernel_grad, c2d.get_kernel_grad())
expected_bias = -0.0064444444444444445
expected_bias_grad = 6.444444444444445
np.testing.assert_almost_equal(expected_bias, c2d.get_bias())
np.testing.assert_almost_equal(expected_bias_grad, c2d.get_bias_grad())
def test_convolution_small(self):
img = np.array([[1, 2, 3, 4], [3, 4, 5, 6], [-1, 0, 1, 3], [0, 2, -1, 4]])
kernel = np.array([[1, -1], [0, 2]])
img_node = node.VarNode('img')
c2d = conv.Convolution2D(img_node, input_shape=(4, 4), kernel=kernel)
var_map = {'img': img}
img_node.forward(var_map)
info("Original x into the convolution layer")
info(repr(img))
output_image = c2d.value()
info("Output of the convolution layer")
expected_output = np.array([[7., 9., 11.],
[-1., 1., 5.],
[3., -3., 6.]])
np.testing.assert_array_almost_equal(expected_output, output_image)
info(repr(output_image))
log_at_info()
c2d.backward(output_image * 0.1, self, var_map)
info("Kernel before gradient descent")
info(repr(c2d.get_kernel()))
def optimizer_function(_w, grad, local_node_storage={}):
return _w - 0.001 * grad
optimizer = core.np.Optimization.OptimizerIterator([img_node], c2d, optimizer_function)
loss = optimizer.step(var_map, np.ones_like(output_image))
info("Printing loss matrix - not really loss but just the output of the last node")
info(repr(loss))
info("Printing kernel after gradient descent")
info(repr(c2d.get_kernel()))
expected_kernel = np.array([[0.998, -1.003111],
[-1.444444444e-3, 1.9973333]])
info("kernel gradient:{}".format(repr(c2d.kernel_grad)))
np.testing.assert_array_almost_equal(expected_kernel, c2d.get_kernel())
self.assertAlmostEqual(-0.001, c2d.get_bias())
info("Bias after gradient descent:{}".format(c2d.get_bias()))
info("Gradient of bias :{}".format(c2d.bias_grad))
def test_linear_training(self):
r"""
For fastest results, use batch size of 64, adam optimizer
and 3 epochs. You should get more than 97% accuracy
:return:
"""
# Build the network
x_node = node.VarNode('x')
yt_node = node.VarNode('yt')
linear1 = node.DenseLayer(x_node,
100,
name="Dense-First",
weight_scale=0.01)
relu1 = act.RelUNode(linear1, name="RelU-First")
linear2 = node.DenseLayer(relu1,
200,
name="Dense-Second",
weight_scale=0.01)
relu2 = act.RelUNode(linear2, name="RelU-Second")
linear3 = node.DenseLayer(relu2,
10,
name="Dense-Third",
weight_scale=0.01)
cross_entropy = loss.LogitsCrossEntropy(linear3, yt_node, name="XEnt")
# Set up optimizers and params
batch_size = 64
epochs = 3
optimizer_func = autodiff_optim.AdamOptimizer()
# optimizer_func = autodiff_optim.SGDOptimizer(lr=.1)
optimizer = autodiff_optim.OptimizerIterator([x_node, yt_node],
cross_entropy,
optimizer_func)
log_at_info()
losses = []
iter_count = 1
predictor = node.make_evaluator([x_node, yt_node], linear3)
ctx = node.ComputeContext({})
mnist = Mnist784()
total_time = time.time()
for epoch in range(epochs):
epoch_time = time.time()
iter = 0
for x, y in mnist.train_iterator_seq(batch_size=batch_size):
ctx['x'], ctx['yt'] = x, y
iter_loss = optimizer.step(ctx, 1.0) / batch_size
losses.append(iter_loss)
iter += 1
if iter % 100 == 0:
print("iter:{}".format(iter))
loss_av = np.array(losses[:-batch_size + 1])
loss_av = np.mean(loss_av)
e, xv, yv = mnist.test_iterator(1, batch_size=-1, one_hot=False)
ctx['x'], ctx['yt'] = xv, yv
percent = self.measure_validation_perf(predictor, ctx, yv)
epoch_time = time.time() - epoch_time
info("Iter {:2d}:: Val:{:2.4f}% , loss av={:01.8f}, time:{:2.3f}s".
format(epoch, percent, loss_av, epoch_time))
total_time = time.time() - total_time
info("Total time taken:{:4.4f}".format(total_time))
# syshelp
elif command == 'help -sys':
core.assist_sys()
# geom help
elif command == 'help -geom':
core.assist_geom()
# math
elif command == 'math':
core.climath()
# apache
elif command == 'info':
core.info()
# right triangle checker
elif command == 'rightcheck':
core.rightcheck()
# quadratic formula
elif command == 'quadratic':
core.quadratic()
# find parts of right triangle
elif command == 'pythagorean':
core.pythagorean()
# find midpoint
elif command == 'midpoint':
def test_forward(self):
self.forward()
value = self.max_pool_node.value()
info("maxpool = np.{}".format(repr(value)))
expected_value = np.array([[2, 3, 4], [9, 9, 5]])
np.testing.assert_almost_equal(expected_value, value)
