def _get_prediction(self, x, name=None, verbose=None):
if verbose is None:
verbose = self.verbose
fc_shape = np.prod(x.shape[1:]) # type: int
single_batch = int(NNConfig.BATCH_SIZE / fc_shape)
if not single_batch:
single_batch = 1
if not single_batch:
single_batch = 1
if single_batch >= len(x):
return self._sess.run(self._y_pred, {self._tfx: x})
epoch = int(len(x) / single_batch)
if not len(x) % single_batch:
epoch += 1
name = "Prediction" if name is None else "Prediction ({})".format(name)
sub_bar = ProgressBar(max_value=epoch, name=name, start=False)
if verbose >= NNVerbose.METRICS:
sub_bar.start()
rs, count = [], 0
while count < len(x):
count += single_batch
if count >= len(x):
rs.append(self._sess.run(self._y_pred, {self._tfx: x[count - single_batch:]}))
else:
rs.append(self._sess.run(self._y_pred, {self._tfx: x[count - single_batch:count]}))
if verbose >= NNVerbose.METRICS:
sub_bar.update()
return np.vstack(rs)
def fit(self, x, y, sample_weight=None, clf=None, epoch=None, eps=None, **kwargs):
if sample_weight is None:
sample_weight = self._params["sample_weight"]
if clf is None:
clf = self._params["clf"]
if epoch is None:
epoch = self._params["epoch"]
if eps is None:
eps = self._params["eps"]
x, y = np.atleast_2d(x), np.asarray(y)
if clf is None:
clf = "Cart"
kwargs = {"max_depth": 1}
self._kwarg_cache = kwargs
self._clf = clf
if sample_weight is None:
sample_weight = np.ones(len(y)) / len(y)
else:
sample_weight = np.asarray(sample_weight)
bar = ProgressBar(max_value=epoch, name="AdaBoost")
for _ in range(epoch):
tmp_clf = AdaBoost._weak_clf[clf](**kwargs)
tmp_clf.fit(x, y, sample_weight=sample_weight)
y_pred = tmp_clf.predict(x)
em = min(max((y_pred != y).astype(np.int8).dot(sample_weight[..., None])[0], eps), 1 - eps)
am = 0.5 * log(1 / em - 1)
sample_weight *= np.exp(-am * y * y_pred)
sample_weight /= np.sum(sample_weight)
self._clfs.append(deepcopy(tmp_clf))
self._clfs_weights.append(am)
bar.update()
self._clfs_weights = np.array(self._clfs_weights, dtype=np.float32)
def fit(self, x, n_clusters=None, epoch=None, norm=None, animation_params=None):
if n_clusters is None:
n_clusters = self._params["n_clusters"]
if epoch is None:
epoch = self._params["epoch"]
if norm is not None:
self._params["norm"] = norm
*animation_properties, animation_params = self._get_animation_params(animation_params)
x = np.atleast_2d(x)
arange = np.arange(n_clusters)[..., None]
x_high_dim, labels_cache, counter = x[:, None, ...], None, 0
self._centers = x[np.random.permutation(len(x))[:n_clusters]]
bar = ProgressBar(max_value=epoch, name="KMeans")
ims = []
for i in range(epoch):
labels = self.predict(x_high_dim, high_dim=True)
if labels_cache is None:
labels_cache = labels
else:
if np.all(labels_cache == labels):
bar.update(epoch)
break
else:
labels_cache = labels
for j, indices in enumerate(labels == arange):
self._centers[j] = np.average(x[indices], axis=0)
counter += 1
animation_params["extra"] = self._centers
self._handle_animation(i, x, labels, ims, animation_params, *animation_properties)
bar.update()
self._counter = counter
self._handle_mp4(ims, animation_properties)
def _get_prediction(self, x, name=None, batch_size=1e6, verbose=None):
if verbose is None:
verbose = self.verbose
fc_shape = np.prod(x.shape[1:]) # type: int
single_batch = int(batch_size / fc_shape)
if not single_batch:
single_batch = 1
if single_batch >= len(x):
return self._get_activations(x, predict=True).pop()
epoch = int(len(x) / single_batch)
if not len(x) % single_batch:
epoch += 1
name = "Prediction" if name is None else "Prediction ({})".format(name)
sub_bar = ProgressBar(max_value=epoch, name=name, start=False)
if verbose >= NNVerbose.METRICS:
sub_bar.start()
rs, count = [self._get_activations(x[:single_batch], predict=True).pop()], single_batch
if verbose >= NNVerbose.METRICS:
sub_bar.update()
while count < len(x):
count += single_batch
if count >= len(x):
rs.append(self._get_activations(x[count-single_batch:], predict=True).pop())
else:
rs.append(self._get_activations(x[count-single_batch:count], predict=True).pop())
if verbose >= NNVerbose.METRICS:
sub_bar.update()
return np.vstack(rs)
def fit(self, x, y, sample_weight=None, lr=None, epoch=None, animation_params=None):
if sample_weight is None:
sample_weight = self._params["sample_weight"]
if lr is None:
lr = self._params["lr"]
if epoch is None:
epoch = self._params["epoch"]
*animation_properties, animation_params = self._get_animation_params(animation_params)
x, y = np.atleast_2d(x), np.asarray(y)
if sample_weight is None:
sample_weight = np.ones(len(y))
else:
sample_weight = np.asarray(sample_weight) * len(y)
self._w = np.random.random(x.shape[1])
self._b = 0.
ims = []
bar = ProgressBar(max_value=epoch, name="Perceptron")
for i in range(epoch):
y_pred = self.predict(x, True)
err = -y * y_pred * sample_weight
idx = np.argmax(err)
if err[idx] < 0:
bar.terminate()
break
w_norm = np.linalg.norm(self._w)
delta = lr * y[idx] * sample_weight[idx] / w_norm
self._w += delta * (x[idx] - y_pred[idx] * self._w / w_norm ** 2)
self._b += delta
self._handle_animation(i, x, y, ims, animation_params, *animation_properties)
bar.update()
self._handle_mp4(ims, animation_properties)
def run(clf):
acc_records, y_records = [], []
bar = ProgressBar(max_value=10, name="Main")
for _ in range(10):
if clf == "Naive Bayes":
_clf = SKMultinomialNB(alpha=0.1)
elif clf == "Non-linear SVM":
_clf = SKSVM()
else:
_clf = SKLinearSVM()
rs = main(_clf)
acc_records.append(rs[0])
y_records += rs[1]
bar.update()
acc_records = np.array(acc_records) * 100
plt.figure()
plt.boxplot(acc_records, vert=False, showmeans=True)
plt.show()
from Util.DataToolkit import DataToolkit
idx = np.argmax(acc_records) # type: int
print(metrics.classification_report(y_records[idx][0], y_records[idx][1], target_names=np.load(os.path.join(
"_Data", "LABEL_DIC.npy"
))))
toolkit = DataToolkit(acc_records[np.argmax(np.average(acc_records, axis=1))])
print("Acc Mean : {:8.6}".format(toolkit.mean))
print("Acc Variance : {:8.6}".format(toolkit.variance))
print("Done")
def opt(self, epoch=None, eps=None):
"""
Main procedure of opt
:param epoch : Maximum iteration ; default: 1000
:param eps : Tolerance ; default: 1e-8
:return : x*, f*, n_iter, feva
"""
if epoch is None:
epoch = self._params["epoch"]
if eps is None:
eps = self._params["eps"]
self._func.refresh_cache(self._x)
self._loss_cache, self._grad_cache = self.func(0), self.func(1)
bar = ProgressBar(max_value=epoch, name="Opt")
for _ in range(epoch):
self.iter += 1
with warnings.catch_warnings():
warnings.filterwarnings("error")
try:
if self._core(eps):
break
self.log.append(self._loss_cache)
except RuntimeWarning as err:
print("\n", err, "\n")
break
except np.linalg.linalg.LinAlgError as err:
print("\n", err, "\n")
break
bar.update()
bar.update()
bar.terminate()
return self._x, self._loss_cache, self.iter, self.feva
def main(clf):
dat_path = os.path.join("_Data", "dataset.dat")
gen_dataset(dat_path)
with open(dat_path, "rb") as _file:
x, y = pickle.load(_file)
x = [" ".join(sentence) for sentence in x]
_indices = np.random.permutation(len(x))
x = list(np.array(x)[_indices])
y = list(np.array(y)[_indices])
data_len = len(x)
batch_size = math.ceil(data_len * 0.1)
acc_lst, y_results = [], []
bar = ProgressBar(max_value=10, name=str(clf))
for i in range(10):
_next = (i + 1) * batch_size if i != 9 else data_len
x_train = x[:i * batch_size] + x[(i + 1) * batch_size:]
y_train = y[:i * batch_size] + y[(i + 1) * batch_size:]
x_test, y_test = x[i * batch_size:_next], y[i * batch_size:_next]
count_vec = CountVectorizer()
counts_train = count_vec.fit_transform(x_train)
x_test = count_vec.transform(x_test)
tfidf_transformer = TfidfTransformer()
x_train = tfidf_transformer.fit_transform(counts_train)
clf.fit(x_train, y_train)
y_pred = clf.predict(x_test)
acc_lst.append(clf.acc(y_test, y_pred))
y_results.append([y_test, y_pred])
del x_train, y_train, x_test, y_test, y_pred
bar.update()
return acc_lst, y_results
def _get_prediction(self, x, name=None, batch_size=1e6, verbose=None, out_of_sess=False):
if verbose is None:
verbose = self.verbose
fc_shape = np.prod(x.shape[1:]) # type: float
single_batch = int(batch_size / fc_shape)
if not single_batch:
single_batch = 1
if single_batch >= len(x):
if not out_of_sess:
return self._y_pred.eval(feed_dict={self._tfx: x})
with self._sess.as_default():
x = x.astype(np.float32)
return self.get_rs(x).eval(feed_dict={self._tfx: x})
epoch = int(len(x) / single_batch)
if not len(x) % single_batch:
epoch += 1
name = "Prediction" if name is None else "Prediction ({})".format(name)
sub_bar = ProgressBar(max_value=epoch, name=name, start=False)
if verbose >= NNVerbose.METRICS:
sub_bar.start()
if not out_of_sess:
rs = [self._y_pred.eval(feed_dict={self._tfx: x[:single_batch]})]
else:
rs = [self.get_rs(x[:single_batch])]
count = single_batch
if verbose >= NNVerbose.METRICS:
sub_bar.update()
while count < len(x):
count += single_batch
if count >= len(x):
if not out_of_sess:
rs.append(self._y_pred.eval(feed_dict={self._tfx: x[count - single_batch:]}))
else:
rs.append(self.get_rs(x[count - single_batch:]))
else:
if not out_of_sess:
rs.append(self._y_pred.eval(feed_dict={self._tfx: x[count - single_batch:count]}))
else:
rs.append(self.get_rs(x[count - single_batch:count]))
if verbose >= NNVerbose.METRICS:
sub_bar.update()
if out_of_sess:
with self._sess.as_default():
rs = [_rs.eval() for _rs in rs]
return np.vstack(rs)
def fit(self, x, y, sample_weight=None, c=None, lr=None, optimizer=None,
batch_size=None, epoch=None, tol=None, animation_params=None):
if sample_weight is None:
sample_weight = self._params["sample_weight"]
if c is None:
c = self._params["c"]
if lr is None:
lr = self._params["lr"]
if batch_size is None:
batch_size = self._params["batch_size"]
if epoch is None:
epoch = self._params["epoch"]
if tol is None:
tol = self._params["tol"]
if optimizer is None:
optimizer = self._params["optimizer"]
*animation_properties, animation_params = self._get_animation_params(animation_params)
x, y = np.atleast_2d(x), np.asarray(y, dtype=np.float32)
if sample_weight is None:
sample_weight = np.ones(len(y))
else:
sample_weight = np.asarray(sample_weight) * len(y)
self._w = np.zeros(x.shape[1], dtype=np.float32)
self._b = np.zeros(1, dtype=np.float32)
self._model_parameters = [self._w, self._b]
self._optimizer = OptFactory().get_optimizer_by_name(
optimizer, self._model_parameters, lr, epoch
)
bar = ProgressBar(max_value=epoch, name="LinearSVM")
ims = []
train_repeat = self._get_train_repeat(x, batch_size)
for i in range(epoch):
self._optimizer.update()
l = self._batch_training(
x, y, batch_size, train_repeat, sample_weight, c
)
if l < tol:
bar.terminate()
break
self._handle_animation(i, x, y, ims, animation_params, *animation_properties)
bar.update()
self._handle_mp4(ims, animation_properties)
def predict(self, x, get_raw_results=False, **kwargs):
x = NNDist._transfer_x(np.asarray(x))
rs = []
batch_size = floor(1e6 / np.prod(x.shape[1:]))
epoch = int(ceil(len(x) / batch_size))
output = self._sess.graph.get_tensor_by_name(self._output)
bar = ProgressBar(max_value=epoch, name="Predict")
for i in range(epoch):
if i == epoch - 1:
rs.append(self._sess.run(output, {
self._entry: x[i * batch_size:]
}))
else:
rs.append(self._sess.run(output, {
self._entry: x[i * batch_size:(i + 1) * batch_size]
}))
bar.update()
y_pred = np.vstack(rs).astype(np.float32)
return y_pred if get_raw_results else np.argmax(y_pred, axis=1)
def fit(self, x, y, c=None, lr=None, batch_size=None, epoch=None, tol=None,
optimizer=None, animation_params=None):
if c is None:
c = self._params["c"]
if lr is None:
lr = self._params["lr"]
if batch_size is None:
batch_size = self._params["batch_size"]
if epoch is None:
epoch = self._params["epoch"]
if tol is None:
tol = self._params["tol"]
if optimizer is None:
optimizer = self._params["optimizer"]
*animation_properties, animation_params = self._get_animation_params(animation_params)
x, y = np.atleast_2d(x), np.asarray(y, dtype=np.float32)
y_2d = y[..., None]
self._w = Variable(torch.rand([x.shape[1], 1]), requires_grad=True)
self._b = Variable(torch.Tensor([0.]), requires_grad=True)
self._model_parameters = [self._w, self._b]
self._optimizer = PyTorchOptFac().get_optimizer_by_name(
optimizer, self._model_parameters, lr, epoch
)
x, y, y_2d = self._arr_to_variable(False, x, y, y_2d)
loss_function = lambda _y, _y_pred: self._loss(_y, _y_pred, c)
bar = ProgressBar(max_value=epoch, name="TorchLinearSVM")
ims = []
train_repeat = self._get_train_repeat(x, batch_size)
for i in range(epoch):
self._optimizer.update()
l = self.batch_training(
x, y_2d, batch_size, train_repeat, loss_function
)
if l < tol:
bar.terminate()
break
self._handle_animation(i, x, y, ims, animation_params, *animation_properties)
bar.update()
self._handle_mp4(ims, animation_properties)
def predict(self, x):
self._create_graph()
x, rs = np.atleast_2d(x).astype(np.float32), []
with tf.Session() as sess:
flattened_tensor = sess.graph.get_tensor_by_name(self._output)
print("Predicting...")
batch_size = math.floor(1e6 / np.prod(x.shape[1:]))
epoch = math.ceil(len(x) / batch_size) # type: int
bar = ProgressBar(max_value=epoch, name="Predict")
for i in range(epoch):
if i == epoch - 1:
rs.append(sess.run(flattened_tensor, {
self._entry: x[i*batch_size:]
}))
else:
rs.append(sess.run(flattened_tensor, {
self._entry: x[i*batch_size:(i+1)*batch_size]
}))
bar.update()
return np.vstack(rs).astype(np.float32)
def fit(self, x, y, c=None, lr=None, batch_size=None, epoch=None, tol=None,
optimizer=None, animation_params=None):
if c is None:
c = self._params["c"]
if lr is None:
lr = self._params["lr"]
if batch_size is None:
batch_size = self._params["batch_size"]
if epoch is None:
epoch = self._params["epoch"]
if tol is None:
tol = self._params["tol"]
if optimizer is None:
optimizer = self._params["optimizer"]
*animation_properties, animation_params = self._get_animation_params(animation_params)
x, y = np.atleast_2d(x), np.asarray(y)
y_2d = y[..., None]
self._w = tf.Variable(np.zeros([x.shape[1], 1]), dtype=tf.float32, name="w")
self._b = tf.Variable(0., dtype=tf.float32, name="b")
self._tfx = tf.placeholder(tf.float32, [None, x.shape[1]])
self._tfy = tf.placeholder(tf.float32, [None, 1])
self._y_pred_raw = tf.matmul(self._tfx, self._w) + self._b
self._y_pred = tf.sign(self._y_pred_raw)
loss = tf.reduce_sum(
tf.nn.relu(1 - self._tfy * self._y_pred_raw)
) + c * tf.nn.l2_loss(self._w)
train_step = TFOptFac().get_optimizer_by_name(optimizer, lr).minimize(loss)
self._sess.run(tf.global_variables_initializer())
bar = ProgressBar(max_value=epoch, name="TFLinearSVM")
ims = []
train_repeat = self._get_train_repeat(x, batch_size)
for i in range(epoch):
l = self._batch_training(x, y_2d, batch_size, train_repeat, loss, train_step)
if l < tol:
bar.terminate()
break
self._handle_animation(i, x, y, ims, animation_params, *animation_properties)
bar.update()
self._handle_mp4(ims, animation_properties)
def fit(self, x, y, sample_weight=None, c=None, lr=None, epoch=None, tol=None, animation_params=None):
if sample_weight is None:
sample_weight = self._params["sample_weight"]
if c is None:
c = self._params["c"]
if lr is None:
lr = self._params["lr"]
if epoch is None:
epoch = self._params["epoch"]
if tol is None:
tol = self._params["tol"]
*animation_properties, animation_params = self._get_animation_params(animation_params)
x, y = np.atleast_2d(x), np.asarray(y)
if sample_weight is None:
sample_weight = np.ones(len(y))
else:
sample_weight = np.asarray(sample_weight) * len(y)
self._w = np.zeros(x.shape[1])
self._b = 0
ims = []
bar = ProgressBar(max_value=epoch, name="LinearSVM")
for i in range(epoch):
err = (1 - self.predict(x, get_raw_results=True) * y) * sample_weight
indices = np.random.permutation(len(y))
idx = indices[np.argmax(err[indices])]
if err[idx] <= tol:
bar.update(epoch)
break
delta = lr * c * y[idx] * sample_weight[idx]
self._w *= 1 - lr
self._w += delta * x[idx]
self._b += delta
self._handle_animation(i, x, y, ims, animation_params, *animation_properties)
bar.update()
self._handle_mp4(ims, animation_properties)
def fit(self, x, y, sample_weight=None, tree=None, epoch=None, feature_bound=None, **kwargs):
if sample_weight is None:
sample_weight = self._params["sample_weight"]
if tree is None:
tree = self._params["tree"]
if epoch is None:
epoch = self._params["epoch"]
if feature_bound is None:
feature_bound = self._params["feature_bound"]
x, y = np.atleast_2d(x), np.asarray(y)
n_sample = len(y)
self._tree = tree
bar = ProgressBar(max_value=epoch, name="RF")
for _ in range(epoch):
tmp_tree = RandomForest.cvd_trees[tree](**kwargs)
indices = np.random.randint(n_sample, size=n_sample)
if sample_weight is None:
local_weight = None
else:
local_weight = sample_weight[indices]
local_weight /= local_weight.sum()
tmp_tree.fit(x[indices], y[indices], sample_weight=local_weight, feature_bound=feature_bound)
self._trees.append(deepcopy(tmp_tree))
bar.update()
def fit(self,
x=None,
y=None,
x_test=None,
y_test=None,
lr=0.01,
lb=0.01,
epoch=20,
weight_scale=1,
batch_size=256,
record_period=1,
train_only=False,
optimizer=None,
show_loss=True,
metrics=None,
do_log=False,
verbose=None,
visualize=False,
visualize_setting=None,
draw_detailed_network=False,
weight_average=None):
x, y = self._feed_data(x, y)
self._lr = lr
self._init_optimizer(optimizer)
print("Optimizer: ", self._optimizer.name)
print("-" * 30)
if not self._layers:
raise BuildNetworkError(
"Please provide layers before fitting data")
if y.shape[1] != self._current_dimension:
raise BuildNetworkError(
"Output layer's shape should be {}, {} found".format(
self._current_dimension, y.shape[1]))
(x_train, x_test), (y_train,
y_test) = self.split_data(x, y, x_test, y_test,
train_only)
train_len = len(x_train)
batch_size = min(batch_size, train_len)
do_random_batch = train_len >= batch_size
train_repeat = int(train_len / batch_size) + 1
self._feed_data(x_train, y_train)
self._tfx = tf.placeholder(tf.float32, shape=[None, *x.shape[1:]])
self._tfy = tf.placeholder(tf.float32, shape=[None, y.shape[1]])
self._metrics = ["acc"] if metrics is None else metrics
for i, metric in enumerate(self._metrics):
if isinstance(metric, str):
if metric not in self._available_metrics:
raise BuildNetworkError(
"Metric '{}' is not implemented".format(metric))
self._metrics[i] = self._available_metrics[metric]
self._metric_names = [_m.__name__ for _m in self._metrics]
self._logs = {
name: [[] for _ in range(len(self._metrics) + 1)]
for name in ("train", "test")
}
if verbose is not None:
self.verbose = verbose
bar = ProgressBar(min_value=0,
max_value=max(1, epoch // record_period),
name="Epoch")
if self.verbose >= NNVerbose.EPOCH:
bar.start()
img = None
with self._sess.as_default() as sess:
# Session
self._y_pred = self.get_rs(self._tfx)
self._l2_loss = self._get_l2_loss(lb)
self._loss = self.get_rs(self._tfx, self._tfy) + self._l2_loss
self._activations = self._get_activations(self._tfx)
self._init_train_step(sess)
for weight in self._tf_weights:
weight *= weight_scale
# Log
merge_op = tf.summary.merge_all()
writer = tf.summary.FileWriter("Logs", sess.graph)
writer.add_graph(sess.graph)
with tf.name_scope("Global_Summaries"):
tf.summary.scalar("l2 loss", self._l2_loss)
tf.summary.scalar("loss", self._loss)
tf.summary.scalar("lr", self._lr)
sub_bar = ProgressBar(min_value=0,
max_value=train_repeat * record_period - 1,
name="Iteration")
for counter in range(epoch):
if self.verbose >= NNVerbose.EPOCH and counter % record_period == 0:
sub_bar.start()
for i in range(train_repeat):
if do_random_batch:
batch = np.random.choice(train_len, batch_size)
x_batch, y_batch = x_train[batch], y_train[batch]
else:
x_batch, y_batch = x_train, y_train
feed_dict = {self._tfx: x_batch, self._tfy: y_batch}
self._train_step.run(feed_dict=feed_dict)
summary = sess.run(merge_op, feed_dict=feed_dict)
writer.add_summary(summary, counter * train_repeat + i)
if self.verbose >= NNVerbose.DEBUG:
pass
if self.verbose >= NNVerbose.EPOCH:
if sub_bar.update(
) and self.verbose >= NNVerbose.METRICS_DETAIL:
self._append_log(x, y, "train", get_loss=show_loss)
self._append_log(x_test,
y_test,
"test",
get_loss=show_loss)
self._print_metric_logs(show_loss, "train")
self._print_metric_logs(show_loss, "test")
if self.verbose >= NNVerbose.EPOCH:
sub_bar.update()
if (counter + 1) % record_period == 0:
if do_log:
self._append_log(x, y, "train", get_loss=show_loss)
self._append_log(x_test,
y_test,
"test",
get_loss=show_loss)
if self.verbose >= NNVerbose.METRICS:
self._print_metric_logs(show_loss, "train")
self._print_metric_logs(show_loss, "test")
if visualize:
if visualize_setting is None:
self.visualize2d(x_test, y_test)
else:
self.visualize2d(x_test, y_test,
*visualize_setting)
if x_test.shape[1] == 2:
if draw_detailed_network:
img = self._draw_detailed_network(
weight_average=weight_average)
if self.verbose >= NNVerbose.EPOCH:
bar.update(counter // record_period + 1)
sub_bar = ProgressBar(
min_value=0,
max_value=train_repeat * record_period - 1,
name="Iteration")
if img is not None:
cv2.waitKey(0)
cv2.destroyAllWindows()
return self._logs
_prob_lst = prob_lists[i]
x_test, y_test = test_sets[i]
y_pred = np.array(
[pick_best(sentence, _prob_lst) for sentence in x_test])
y_test = np.array(y_test)
acc_lst.append(100 * np.sum(y_pred == y_test) / len(y_pred))
return acc_lst
if __name__ == '__main__':
_rs, epoch = [], 10
bar = ProgressBar(max_value=epoch, name="_NB")
bar.start()
for _ in range(epoch):
_rs.append(test(*train()))
bar.update()
_rs = np.array(_rs).T
# x_base = np.arange(len(_rs[0])) + 1
# plt.figure()
# for _acc_lst in _rs:
# plt.plot(x_base, _acc_lst)
# plt.plot(x_base, np.average(_rs, axis=0), linewidth=4, label="Average")
# plt.xlim(1, epoch)
# plt.ylim(np.min(_rs), np.max(_rs)+2)
# plt.legend(loc="lower right")
# plt.show()
plt.figure()
plt.boxplot(_rs.T, vert=False, showmeans=True)
plt.show()
_rs = np.array(_rs).ravel()
print("Acc Mean : {:8.6}".format(np.average(_rs)))
def fit(self, x=None, y=None, lr=0.01, epoch=10, batch_size=128, train_rate=None,
verbose=0, metrics=None, record_period=100):
# Initialize
self.verbose = verbose
self._add_cost_layer()
self._init_optimizers(lr)
layer_width = len(self._layers)
if train_rate is not None:
train_rate = float(train_rate)
train_len = int(len(x) * train_rate)
shuffle_suffix = np.random.permutation(int(len(x)))
x, y = x[shuffle_suffix], y[shuffle_suffix]
x_train, y_train = x[:train_len], y[:train_len]
x_test, y_test = x[train_len:], y[train_len:]
else:
x_train = x_test = x
y_train = y_test = y
train_len = len(x_train)
batch_size = min(batch_size, train_len)
do_random_batch = train_len >= batch_size
train_repeat = int(train_len / batch_size) + 1
self._metrics = ["acc"] if metrics is None else metrics
for i, metric in enumerate(self._metrics):
if isinstance(metric, str):
self._metrics[i] = self._available_metrics[metric]
self._metric_names = [_m.__name__ for _m in self._metrics]
self._logs = {
name: [[] for _ in range(len(self._metrics) + 1)] for name in ("train", "test")
}
bar = ProgressBar(max_value=max(1, epoch // record_period), name="Epoch", start=False)
if self.verbose >= NNVerbose.EPOCH:
bar.start()
# Train
sub_bar = ProgressBar(max_value=train_repeat * record_period - 1, name="Iteration", start=False)
for counter in range(epoch):
if self.verbose >= NNVerbose.EPOCH and counter % record_period == 0:
sub_bar.start()
for _ in range(train_repeat):
if do_random_batch:
batch = np.random.choice(train_len, batch_size)
x_batch, y_batch = x_train[batch], y_train[batch]
else:
x_batch, y_batch = x_train, y_train
self._w_optimizer.update(); self._b_optimizer.update()
_activations = self._get_activations(x_batch)
_deltas = [self._layers[-1].bp_first(y_batch, _activations[-1])]
for i in range(-1, -len(_activations), -1):
_deltas.append(
self._layers[i - 1].bp(_activations[i - 1], self._weights[i], _deltas[-1])
)
for i in range(layer_width - 2, 0, -1):
self._opt(i, _activations[i - 1], _deltas[layer_width - i - 1])
self._opt(0, x_batch, _deltas[-1])
if self.verbose >= NNVerbose.EPOCH:
if sub_bar.update() and self.verbose >= NNVerbose.METRICS_DETAIL:
self._append_log(x_train, y_train, "train")
self._append_log(x_test, y_test, "test")
self._print_metric_logs("train")
self._print_metric_logs("test")
if self.verbose >= NNVerbose.EPOCH:
sub_bar.update()
if (counter + 1) % record_period == 0:
self._append_log(x_train, y_train, "train")
self._append_log(x_test, y_test, "test")
if self.verbose >= NNVerbose.METRICS:
self._print_metric_logs("train")
self._print_metric_logs("test")
if self.verbose >= NNVerbose.EPOCH:
bar.update(counter // record_period + 1)
sub_bar = ProgressBar(max_value=train_repeat * record_period - 1, name="Iteration", start=False)
def fit(self,
x=None, y=None, x_test=None, y_test=None,
batch_size=256, record_period=1, train_only=False,
optimizer=None, w_optimizer=None, b_optimizer=None,
lr=0.01, lb=0.01, epoch=20, weight_scale=1, apply_bias=True,
show_loss=True, metrics=None, do_log=True, verbose=None,
visualize=False, visualize_setting=None,
draw_weights=False, draw_network=False, draw_detailed_network=False,
draw_img_network=False, img_shape=None, weight_average=None):
if draw_img_network and img_shape is None:
raise BuildNetworkError("Please provide image's shape to draw_img_network")
x, y = self._feed_data(x, y)
self._lr, self._epoch = lr, epoch
for weight in self._weights:
weight *= weight_scale
if not self._w_optimizer or not self._b_optimizer:
if not self._optimizer_name:
if optimizer is None:
optimizer = "Adam"
self._w_optimizer = optimizer if w_optimizer is None else w_optimizer
self._b_optimizer = optimizer if b_optimizer is None else b_optimizer
else:
if not self._w_optimizer:
self._w_optimizer = self._optimizer_name
if not self._b_optimizer:
self._b_optimizer = self._optimizer_name
self._init_optimizer()
assert isinstance(self._w_optimizer, Optimizer) and isinstance(self._b_optimizer, Optimizer)
print()
print("=" * 30)
print("Optimizers")
print("-" * 30)
print("w: {}\nb: {}".format(self._w_optimizer, self._b_optimizer))
print("-" * 30)
if not self._layers:
raise BuildNetworkError("Please provide layers before fitting data")
if y.shape[1] != self._current_dimension:
raise BuildNetworkError("Output layer's shape should be {}, {} found".format(
self._current_dimension, y.shape[1]))
(x_train, x_test), (y_train, y_test) = self.split_data(
x, y, x_test, y_test, train_only)
train_len = len(x_train)
batch_size = min(batch_size, train_len)
do_random_batch = train_len >= batch_size
train_repeat = int(train_len / batch_size) + 1
self._regularization_param = 1 - lb * lr / batch_size
self._feed_data(x_train, y_train)
self._metrics = ["acc"] if metrics is None else metrics
for i, metric in enumerate(self._metrics):
if isinstance(metric, str):
if metric not in self._available_metrics:
raise BuildNetworkError("Metric '{}' is not implemented".format(metric))
self._metrics[i] = self._available_metrics[metric]
self._metric_names = [_m.__name__ for _m in self._metrics]
self._logs = {
name: [[] for _ in range(len(self._metrics) + 1)] for name in ("train", "cv", "test")
}
if verbose is not None:
self.verbose = verbose
layer_width = len(self._layers)
self._whether_apply_bias = apply_bias
bar = ProgressBar(min_value=0, max_value=max(1, epoch // record_period), name="Epoch")
if self.verbose >= NNVerbose.EPOCH:
bar.start()
img = None
weight_trace = [[[org] for org in weight] for weight in self._weights]
sub_bar = ProgressBar(min_value=0, max_value=train_repeat * record_period - 1, name="Iteration")
for counter in range(epoch):
self._w_optimizer.update()
self._b_optimizer.update()
_xs, _activations = [], []
if self.verbose >= NNVerbose.EPOCH and counter % record_period == 0:
sub_bar.start()
for _i in range(train_repeat):
if do_random_batch:
batch = np.random.choice(train_len, batch_size)
x_batch, y_batch = x_train[batch], y_train[batch]
else:
x_batch, y_batch = x_train, y_train
_activations = self._get_activations(x_batch)
if self.verbose >= NNVerbose.DEBUG:
_xs = [x_batch.dot(self._weights[0])]
for i, weight in enumerate(self._weights[1:]):
_xs.append(_activations[i].dot(weight))
_deltas = [self._layers[-1].bp_first(y_batch, _activations[-1])]
for i in range(-1, -len(_activations), -1):
_deltas.append(self._layers[i - 1].bp(_activations[i - 1], self._weights[i], _deltas[-1]))
for i in range(layer_width - 1, 0, -1):
if not isinstance(self._layers[i], SubLayer):
self._opt(i, _activations[i - 1], _deltas[layer_width - i - 1])
self._opt(0, x_batch, _deltas[-1])
if draw_weights:
for i, weight in enumerate(self._weights):
for j, new_weight in enumerate(weight.copy()):
weight_trace[i][j].append(new_weight)
if self.verbose >= NNVerbose.DEBUG:
print("")
print("## Activations ##")
for i, ac in enumerate(_activations):
print("-- Layer {} ({}) --".format(i + 1, self._layers[i].name))
print(_xs[i])
print(ac)
print("")
print("## Deltas ##")
for i, delta in zip(range(len(_deltas) - 1, -1, -1), _deltas):
print("-- Layer {} ({}) --".format(i + 1, self._layers[i].name))
print(delta)
_ = input("Press any key to continue...")
if self.verbose >= NNVerbose.EPOCH:
if sub_bar.update() and self.verbose >= NNVerbose.METRICS_DETAIL:
self._append_log(x, y, "train", get_loss=show_loss)
self._append_log(x_test, y_test, "cv", get_loss=show_loss)
self._print_metric_logs(show_loss, "train")
self._print_metric_logs(show_loss, "cv")
if self.verbose >= NNVerbose.EPOCH:
sub_bar.update()
if do_log:
self._append_log(x, y, "train", get_loss=show_loss)
self._append_log(x_test, y_test, "cv", get_loss=show_loss)
if (counter + 1) % record_period == 0:
if do_log and self.verbose >= NNVerbose.METRICS:
self._print_metric_logs(show_loss, "train")
self._print_metric_logs(show_loss, "cv")
if visualize:
if visualize_setting is None:
self.visualize2d(x_test, y_test)
else:
self.visualize2d(x_test, y_test, *visualize_setting)
if x_test.shape[1] == 2:
if draw_network:
img = self._draw_network(weight_average=weight_average, activations=_activations)
if draw_detailed_network:
img = self._draw_detailed_network(weight_average=weight_average)
elif draw_img_network:
img = self._draw_img_network(img_shape, weight_average=weight_average)
if self.verbose >= NNVerbose.EPOCH:
bar.update(counter // record_period + 1)
sub_bar = ProgressBar(min_value=0, max_value=train_repeat * record_period - 1, name="Iteration")
if do_log:
self._append_log(x_test, y_test, "test", get_loss=show_loss)
if img is not None:
cv2.waitKey(0)
cv2.destroyAllWindows()
if draw_weights:
ts = np.arange(epoch * train_repeat + 1)
for i, weight in enumerate(self._weights):
plt.figure()
for j in range(len(weight)):
plt.plot(ts, weight_trace[i][j])
plt.title("Weights toward layer {} ({})".format(i + 1, self._layers[i].name))
plt.show()
return self._logs
def fit(self, im, om, generator, cell=None, provide_sequence_length=None,
squeeze=None, use_sparse_labels=None, embedding_size=None, use_final_state=None,
n_hidden=None, n_history=None, activation=None, lr=None, epoch=None, n_iter=None,
batch_size=None, optimizer=None, eps=None, verbose=None):
if cell is None:
cell = self._params["cell"]
if provide_sequence_length is None:
provide_sequence_length = self._params["provide_sequence_length"]
if n_hidden is None:
n_hidden = self._params["n_hidden"]
if n_history is None:
n_history = self._params["n_history"]
if squeeze:
self._squeeze = True
if use_sparse_labels:
self._use_sparse_labels = True
if self._squeeze and n_history == 0:
n_history = 1
if embedding_size:
self._embedding_size = embedding_size
if use_final_state:
self._use_final_state = True
if activation is None:
activation = self._params["activation"]
if lr is None:
lr = self._params["lr"]
if epoch is None:
epoch = self._params["epoch"]
if n_iter is None:
n_iter = self._params["n_iter"]
if optimizer is None:
optimizer = self._params["optimizer"]
if batch_size is None:
batch_size = self._params["batch_size"]
if eps is None:
eps = self._params["eps"]
if verbose is None:
verbose = self._params["verbose"]
self._generator = generator
self._im, self._om, self._activation = im, om, activation
self._optimizer = OptFactory().get_optimizer_by_name(optimizer, lr)
self._define_input(im, om)
self._cell = cell(n_hidden)
self._prepare_for_dynamic_rnn(provide_sequence_length)
rnn_outputs, rnn_final_state = tf.nn.dynamic_rnn(
self._cell, self._input, return_all_states=True,
sequence_length=self._sequence_lengths, initial_state=self._initial_state
)
self._get_output(rnn_outputs, rnn_final_state, n_history)
loss = self._get_loss(eps)
train_step = self._optimizer.minimize(loss)
self._log["iter_err"] = []
self._log["epoch_err"] = []
self._sess.run(tf.global_variables_initializer())
bar = ProgressBar(max_value=epoch, name="Epoch", start=False)
if verbose >= 2:
bar.start()
for _ in range(epoch):
epoch_err = 0
sub_bar = ProgressBar(max_value=n_iter, name="Iter", start=False)
if verbose >= 2:
sub_bar.start()
for __ in range(n_iter):
if provide_sequence_length:
x_batch, y_batch, sequence_length = self._generator.gen(batch_size)
feed_dict = {
self._tfx: x_batch, self._tfy: y_batch,
self._sequence_lengths: sequence_length
}
else:
x_batch, y_batch = self._generator.gen(batch_size)
feed_dict = {self._tfx: x_batch, self._tfy: y_batch}
iter_err = self._sess.run([loss, train_step], feed_dict)[0]
self._log["iter_err"].append(iter_err)
epoch_err += iter_err
if verbose >= 2:
sub_bar.update()
self._log["epoch_err"].append(epoch_err / n_iter)
if verbose >= 1:
self._verbose()
if verbose >= 2:
bar.update()
def fit(self,
x,
y,
lr=None,
epoch=None,
batch_size=None,
train_rate=None,
optimizer=None,
metrics=None,
record_period=None,
verbose=None,
preview=None):
if lr is None:
lr = self._params["lr"]
if epoch is None:
epoch = self._params["epoch"]
if optimizer is None:
optimizer = self._params["optimizer"]
if batch_size is None:
batch_size = self._params["batch_size"]
if train_rate is None:
train_rate = self._params["train_rate"]
if metrics is None:
metrics = self._params["metrics"]
if record_period is None:
record_period = self._params["record_period"]
if verbose is None:
verbose = self._params["verbose"]
if preview is None:
preview = self._params["preview"]
x = NN._transfer_x(x)
self.verbose = verbose
self._optimizer = OptFactory().get_optimizer_by_name(optimizer, lr)
self._tfx = tf.placeholder(tf.float32, shape=[None, *x.shape[1:]])
self._tfy = tf.placeholder(tf.float32, shape=[None, y.shape[1]])
if train_rate is not None:
train_rate = float(train_rate)
train_len = int(len(x) * train_rate)
shuffle_suffix = np.random.permutation(int(len(x)))
x, y = x[shuffle_suffix], y[shuffle_suffix]
x_train, y_train = x[:train_len], y[:train_len]
x_test, y_test = x[train_len:], y[train_len:]
else:
x_train = x_test = x
y_train = y_test = y
y_train_classes = np.argmax(y_train, axis=1)
y_test_classes = np.argmax(y_test, axis=1)
train_len = len(x_train)
batch_size = min(batch_size, train_len)
do_random_batch = train_len >= batch_size
train_repeat, record_period = int(train_len / batch_size) + 1, min(
record_period, epoch)
if metrics is None:
metrics = []
self._metrics = self.get_metrics(metrics)
self._metric_names = [_m.__name__ for _m in metrics]
self._logs = {
name: [[] for _ in range(len(metrics) + 1)]
for name in ("train", "test")
}
bar = ProgressBar(min_value=0,
max_value=max(1, epoch // record_period),
name="Epoch")
if self.verbose >= NNVerbose.EPOCH:
bar.start()
if preview:
self._preview()
with self._sess.as_default() as sess:
self._y_pred = self._get_rs(self._tfx, predict=False)
self._cost = self._layers[-1].calculate(self._tfy, self._y_pred)
self._train_step = self._optimizer.minimize(self._cost)
sess.run(tf.global_variables_initializer())
sub_bar = ProgressBar(min_value=0,
max_value=train_repeat * record_period - 1,
name="Iteration")
for counter in range(epoch):
if self.verbose >= NNVerbose.EPOCH and counter % record_period == 0:
sub_bar.start()
for _i in range(train_repeat):
if do_random_batch:
batch = np.random.choice(train_len, batch_size)
x_batch, y_batch = x_train[batch], y_train[batch]
else:
x_batch, y_batch = x_train, y_train
self._train_step.run(feed_dict={
self._tfx: x_batch,
self._tfy: y_batch
})
if self.verbose >= NNVerbose.EPOCH:
if sub_bar.update(
) and self.verbose >= NNVerbose.METRICS_DETAIL:
self._append_log(x_train, y_train, y_train_classes,
"train")
self._append_log(x_test, y_test, y_test_classes,
"test")
self._print_metric_logs("train")
self._print_metric_logs("test")
if self.verbose >= NNVerbose.EPOCH:
sub_bar.update()
if (counter + 1) % record_period == 0:
if self.verbose >= NNVerbose.METRICS:
self._append_log(x_train, y_train, y_train_classes,
"train")
self._append_log(x_test, y_test, y_test_classes,
"test")
self._print_metric_logs("train")
self._print_metric_logs("test")
if self.verbose >= NNVerbose.EPOCH:
bar.update(counter // record_period + 1)
sub_bar = ProgressBar(
min_value=0,
max_value=train_repeat * record_period - 1,
name="Iteration")
def fit(self, x, y, sample_weight=None, kernel=None, epoch=None,
x_test=None, y_test=None, metrics=None, animation_params=None, **kwargs):
if sample_weight is None:
sample_weight = self._params["sample_weight"] # type: list
if kernel is None:
kernel = self._params["kernel"]
if epoch is None:
epoch = self._params["epoch"]
if x_test is None:
x_test = self._params["x_test"] # type: list
if y_test is None:
y_test = self._params["y_test"] # type: list
if metrics is None:
metrics = self._params["metrics"] # type: list
*animation_properties, animation_params = self._get_animation_params(animation_params)
self._x, self._y = np.atleast_2d(x), np.asarray(y)
if kernel == "poly":
_p = kwargs.get("p", self._params["p"])
self._kernel_name = "Polynomial"
self._kernel_param = "degree = {}".format(_p)
self._kernel = lambda _x, _y: KernelBase._poly(_x, _y, _p)
elif kernel == "rbf":
_gamma = kwargs.get("gamma", 1 / self._x.shape[1])
self._kernel_name = "RBF"
self._kernel_param = r"$\gamma = {:8.6}$".format(_gamma)
self._kernel = lambda _x, _y: KernelBase._rbf(_x, _y, _gamma)
else:
raise NotImplementedError("Kernel '{}' has not defined".format(kernel))
if sample_weight is None:
sample_weight = np.ones(len(y))
else:
sample_weight = np.asarray(sample_weight) * len(y)
self._alpha, self._w, self._prediction_cache = (
np.zeros(len(x)), np.zeros(len(x)), np.zeros(len(x)))
self._gram = self._kernel(self._x, self._x)
self._b = 0
self._prepare(sample_weight, **kwargs)
fit_args, logs, ims = [], [], []
for name, arg in zip(self._fit_args_names, self._fit_args):
if name in kwargs:
arg = kwargs[name]
fit_args.append(arg)
if self._do_log:
if metrics is not None:
self.get_metrics(metrics)
test_gram = None
if x_test is not None and y_test is not None:
x_cv, y_cv = np.atleast_2d(x_test), np.asarray(y_test)
test_gram = self._kernel(self._x, x_cv)
else:
x_cv, y_cv = self._x, self._y
else:
y_cv = test_gram = None
if self._is_torch:
y_cv, self._x, self._y = self._torch_transform(y_cv)
bar = ProgressBar(max_value=epoch, name=str(self))
for i in range(epoch):
if self._fit(sample_weight, *fit_args):
bar.terminate()
break
if self._do_log and metrics is not None:
local_logs = []
for metric in metrics:
if test_gram is None:
if self._is_torch:
local_y = self._y.data.numpy()
else:
local_y = self._y
local_logs.append(metric(local_y, np.sign(self._prediction_cache)))
else:
if self._is_torch:
local_y = y_cv.data.numpy()
else:
local_y = y_cv
local_logs.append(metric(local_y, self.predict(test_gram, gram_provided=True)))
logs.append(local_logs)
self._handle_animation(i, self._x, self._y, ims, animation_params, *animation_properties)
bar.update()
self._handle_mp4(ims, animation_properties)
return logs
def fit(self, x, y, lr=0.01, epoch=10, batch_size=256, train_rate=None,
optimizer="Adam", metrics=None, record_period=100, verbose=1):
self.verbose = verbose
self._add_cost_layer()
self._init_optimizers(optimizer, lr, epoch)
layer_width = len(self._layers)
self._preview()
if train_rate is not None:
train_rate = float(train_rate)
train_len = int(len(x) * train_rate)
shuffle_suffix = np.random.permutation(len(x))
x, y = x[shuffle_suffix], y[shuffle_suffix]
x_train, y_train = x[:train_len], y[:train_len]
x_test, y_test = x[train_len:], y[train_len:]
else:
x_train = x_test = x
y_train = y_test = y
y_train_classes = np.argmax(y_train, axis=1)
y_test_classes = np.argmax(y_test, axis=1)
train_len = len(x_train)
batch_size = min(batch_size, train_len)
do_random_batch = train_len >= batch_size
train_repeat = int(train_len / batch_size) + 1
if metrics is None:
metrics = []
self._metrics = self.get_metrics(metrics)
self._metric_names = [_m.__name__ for _m in metrics]
self._logs = {
name: [[] for _ in range(len(metrics) + 1)] for name in ("train", "test")
}
bar = ProgressBar(min_value=0, max_value=max(1, epoch // record_period), name="Epoch")
if self.verbose >= NNVerbose.EPOCH:
bar.start()
sub_bar = ProgressBar(min_value=0, max_value=train_repeat * record_period - 1, name="Iteration")
for counter in range(epoch):
if self.verbose >= NNVerbose.EPOCH and counter % record_period == 0:
sub_bar.start()
for _ in range(train_repeat):
if do_random_batch:
batch = np.random.choice(train_len, batch_size)
x_batch, y_batch = x_train[batch], y_train[batch]
else:
x_batch, y_batch = x_train, y_train
self._w_optimizer.update()
self._b_optimizer.update()
_activations = self._get_activations(x_batch)
_deltas = [self._layers[-1].bp_first(y_batch, _activations[-1])]
for i in range(-1, -len(_activations), -1):
_deltas.append(
self._layers[i - 1].bp(_activations[i - 1], self._weights[i], _deltas[-1])
)
for i in range(layer_width - 2, 0, -1):
self._opt(i, _activations[i - 1], _deltas[layer_width - i - 1])
self._opt(0, x_batch, _deltas[-1])
if self.verbose >= NNVerbose.EPOCH:
if sub_bar.update() and self.verbose >= NNVerbose.METRICS_DETAIL:
self._append_log(x_train, y_train, y_train_classes, "train")
self._append_log(x_test, y_test, y_test_classes, "test")
self._print_metric_logs("train")
self._print_metric_logs("test")
if self.verbose >= NNVerbose.EPOCH:
sub_bar.update()
if (counter + 1) % record_period == 0:
self._append_log(x_train, y_train, y_train_classes, "train")
self._append_log(x_test, y_test, y_test_classes, "test")
if self.verbose >= NNVerbose.METRICS:
self._print_metric_logs("train")
self._print_metric_logs("test")
if self.verbose >= NNVerbose.EPOCH:
bar.update(counter // record_period + 1)
sub_bar = ProgressBar(min_value=0, max_value=train_repeat * record_period - 1, name="Iteration")
def fit(self,
x,
y,
sample_weight=None,
kernel=None,
epoch=None,
x_test=None,
y_test=None,
metrics=None,
animation_params=None,
**kwargs):
if sample_weight is None:
sample_weight = self._params["sample_weight"] # type: list
if kernel is None:
kernel = self._params["kernel"]
if epoch is None:
epoch = self._params["epoch"]
if x_test is None:
x_test = self._params["x_test"] # type: list
if y_test is None:
y_test = self._params["y_test"] # type: list
if metrics is None:
metrics = self._params["metrics"] # type: list
*animation_properties, animation_params = self._get_animation_params(
animation_params)
self._x, self._y = np.atleast_2d(x), np.asarray(y)
if kernel == "poly":
_p = kwargs.get("p", self._params["p"])
self._kernel_name = "Polynomial"
self._kernel_param = "degree = {}".format(_p)
self._kernel = lambda _x, _y: KernelBase._poly(_x, _y, _p)
elif kernel == "rbf":
_gamma = kwargs.get("gamma", 1 / self._x.shape[1])
self._kernel_name = "RBF"
self._kernel_param = r"$\gamma = {:8.6}$".format(_gamma)
self._kernel = lambda _x, _y: KernelBase._rbf(_x, _y, _gamma)
else:
raise NotImplementedError(
"Kernel '{}' has not defined".format(kernel))
if sample_weight is None:
sample_weight = np.ones(len(y))
else:
sample_weight = np.asarray(sample_weight) * len(y)
self._alpha, self._w, self._prediction_cache = (np.zeros(len(x)),
np.zeros(len(x)),
np.zeros(len(x)))
self._gram = self._kernel(self._x, self._x)
self._b = 0
self._prepare(sample_weight, **kwargs)
fit_args, logs, ims = [], [], []
for name, arg in zip(self._fit_args_names, self._fit_args):
if name in kwargs:
arg = kwargs[name]
fit_args.append(arg)
if self._do_log:
if metrics is not None:
self.get_metrics(metrics)
test_gram = None
if x_test is not None and y_test is not None:
x_cv, y_cv = np.atleast_2d(x_test), np.asarray(y_test)
test_gram = self._kernel(self._x, x_cv)
else:
x_cv, y_cv = self._x, self._y
else:
y_cv = test_gram = None
if self._is_torch:
y_cv, self._x, self._y = self._torch_transform(y_cv)
bar = ProgressBar(max_value=epoch, name=str(self))
for i in range(epoch):
if self._fit(sample_weight, *fit_args):
bar.terminate()
break
if self._do_log and metrics is not None:
local_logs = []
for metric in metrics:
if test_gram is None:
if self._is_torch:
local_y = self._y.data.numpy()
else:
local_y = self._y
local_logs.append(
metric(local_y, np.sign(self._prediction_cache)))
else:
if self._is_torch:
local_y = y_cv.data.numpy()
else:
local_y = y_cv
local_logs.append(
metric(local_y,
self.predict(test_gram,
gram_provided=True)))
logs.append(local_logs)
self._handle_animation(i, self._x, self._y, ims, animation_params,
*animation_properties)
bar.update()
self._handle_mp4(ims, animation_properties)
return logs
def fit(self, x=None, y=None, lr=0.01, epoch=10, batch_size=128, train_rate=None,
verbose=0, metrics=None, record_period=100):
# Initialize
self.verbose = verbose
self._add_cost_layer()
self._init_optimizers(lr)
layer_width = len(self._layers)
if train_rate is not None:
train_rate = float(train_rate)
train_len = int(len(x) * train_rate)
shuffle_suffix = np.random.permutation(int(len(x)))
x, y = x[shuffle_suffix], y[shuffle_suffix]
x_train, y_train = x[:train_len], y[:train_len]
x_test, y_test = x[train_len:], y[train_len:]
else:
x_train = x_test = x
y_train = y_test = y
train_len = len(x_train)
batch_size = min(batch_size, train_len)
do_random_batch = train_len >= batch_size
train_repeat = int(train_len / batch_size) + 1
self._metrics = ["acc"] if metrics is None else metrics
for i, metric in enumerate(self._metrics):
if isinstance(metric, str):
self._metrics[i] = self._available_metrics[metric]
self._metric_names = [_m.__name__ for _m in self._metrics]
self._logs = {
name: [[] for _ in range(len(self._metrics) + 1)] for name in ("train", "test")
}
bar = ProgressBar(max_value=max(1, epoch // record_period), name="Epoch", start=False)
if self.verbose >= NNVerbose.EPOCH:
bar.start()
# Train
sub_bar = ProgressBar(max_value=train_repeat * record_period - 1, name="Iteration", start=False)
for counter in range(epoch):
if self.verbose >= NNVerbose.EPOCH and counter % record_period == 0:
sub_bar.start()
for _ in range(train_repeat):
if do_random_batch:
batch = np.random.choice(train_len, batch_size)
x_batch, y_batch = x_train[batch], y_train[batch]
else:
x_batch, y_batch = x_train, y_train
self._w_optimizer.update(); self._b_optimizer.update()
_activations = self._get_activations(x_batch)
_deltas = [self._layers[-1].bp_first(y_batch, _activations[-1])]
for i in range(-1, -len(_activations), -1):
_deltas.append(
self._layers[i - 1].bp(_activations[i - 1], self._weights[i], _deltas[-1])
)
for i in range(layer_width - 2, 0, -1):
self._opt(i, _activations[i - 1], _deltas[layer_width - i - 1])
self._opt(0, x_batch, _deltas[-1])
if self.verbose >= NNVerbose.EPOCH:
if sub_bar.update() and self.verbose >= NNVerbose.METRICS_DETAIL:
self._append_log(x_train, y_train, "train")
self._append_log(x_test, y_test, "test")
self._print_metric_logs("train")
self._print_metric_logs("test")
if self.verbose >= NNVerbose.EPOCH:
sub_bar.update()
if (counter + 1) % record_period == 0:
self._append_log(x_train, y_train, "train")
self._append_log(x_test, y_test, "test")
if self.verbose >= NNVerbose.METRICS:
self._print_metric_logs("train")
self._print_metric_logs("test")
if self.verbose >= NNVerbose.EPOCH:
bar.update(counter // record_period + 1)
sub_bar = ProgressBar(max_value=train_repeat * record_period - 1, name="Iteration", start=False)
def fit(self,
x=None, y=None, x_test=None, y_test=None,
batch_size=128, record_period=1, train_only=False,
optimizer=None, w_optimizer=None, b_optimizer=None,
lr=0.001, lb=0.001, epoch=20, weight_scale=1, apply_bias=True,
show_loss=True, metrics=None, do_log=True, verbose=None,
visualize=False, visualize_setting=None,
draw_weights=False, animation_params=None):
self._lr, self._epoch = lr, epoch
for weight in self._weights:
weight *= weight_scale
if not self._w_optimizer or not self._b_optimizer:
if not self._optimizer_name:
if optimizer is None:
optimizer = "Adam"
self._w_optimizer = optimizer if w_optimizer is None else w_optimizer
self._b_optimizer = optimizer if b_optimizer is None else b_optimizer
else:
if not self._w_optimizer:
self._w_optimizer = self._optimizer_name
if not self._b_optimizer:
self._b_optimizer = self._optimizer_name
self._init_optimizer()
assert isinstance(self._w_optimizer, Optimizer) and isinstance(self._b_optimizer, Optimizer)
print()
print("=" * 30)
print("Optimizers")
print("-" * 30)
print("w: {}\nb: {}".format(self._w_optimizer, self._b_optimizer))
print("-" * 30)
if not self._layers:
raise BuildNetworkError("Please provide layers before fitting data")
if y.shape[1] != self._current_dimension:
raise BuildNetworkError("Output layer's shape should be {}, {} found".format(
self._current_dimension, y.shape[1]))
(x_train, x_test), (y_train, y_test) = self._split_data(
x, y, x_test, y_test, train_only)
train_len = len(x_train)
batch_size = min(batch_size, train_len)
do_random_batch = train_len > batch_size
train_repeat = 1 if not do_random_batch else int(train_len / batch_size) + 1
self._regularization_param = 1 - lb * lr / batch_size
self._get_min_max(x_train, y_train)
self._metrics = ["acc"] if metrics is None else metrics
for i, metric in enumerate(self._metrics):
if isinstance(metric, str):
if metric not in self._available_metrics:
raise BuildNetworkError("Metric '{}' is not implemented".format(metric))
self._metrics[i] = self._available_metrics[metric]
self._metric_names = [_m.__name__ for _m in self._metrics]
self._logs = {
name: [[] for _ in range(len(self._metrics) + 1)] for name in ("train", "cv", "test")
}
if verbose is not None:
self.verbose = verbose
layer_width = len(self._layers)
self._apply_bias = apply_bias
bar = ProgressBar(max_value=max(1, epoch // record_period), name="Epoch", start=False)
if self.verbose >= NNVerbose.EPOCH:
bar.start()
img, ims = None, []
if draw_weights:
weight_trace = [[[org] for org in weight] for weight in self._weights]
else:
weight_trace = []
*animation_properties, animation_params = self._get_animation_params(animation_params)
sub_bar = ProgressBar(max_value=train_repeat * record_period - 1, name="Iteration", start=False)
for counter in range(epoch):
self._w_optimizer.update()
self._b_optimizer.update()
if self.verbose >= NNVerbose.ITER and counter % record_period == 0:
sub_bar.start()
for _ in range(train_repeat):
if do_random_batch:
batch = np.random.choice(train_len, batch_size)
x_batch, y_batch = x_train[batch], y_train[batch]
else:
x_batch, y_batch = x_train, y_train
activations = self._get_activations(x_batch)
deltas = [self._layers[-1].bp_first(y_batch, activations[-1])]
for i in range(-1, -len(activations), -1):
deltas.append(self._layers[i - 1].bp(activations[i - 1], self._weights[i], deltas[-1]))
for i in range(layer_width - 1, 0, -1):
if not isinstance(self._layers[i], SubLayer):
self._opt(i, activations[i - 1], deltas[layer_width - i - 1])
self._opt(0, x_batch, deltas[-1])
if draw_weights:
for i, weight in enumerate(self._weights):
for j, new_weight in enumerate(weight.copy()):
weight_trace[i][j].append(new_weight)
if self.verbose >= NNVerbose.DEBUG:
pass
if self.verbose >= NNVerbose.ITER:
if sub_bar.update() and self.verbose >= NNVerbose.METRICS_DETAIL:
self._append_log(x, y, "train", get_loss=show_loss)
self._append_log(x_test, y_test, "cv", get_loss=show_loss)
self._print_metric_logs(show_loss, "train")
self._print_metric_logs(show_loss, "cv")
if self.verbose >= NNVerbose.ITER:
sub_bar.update()
self._handle_animation(
counter, x, y, ims, animation_params, *animation_properties,
img=self._draw_2d_network(**animation_params), name="Neural Network"
)
if do_log:
self._append_log(x, y, "train", get_loss=show_loss)
self._append_log(x_test, y_test, "cv", get_loss=show_loss)
if (counter + 1) % record_period == 0:
if do_log and self.verbose >= NNVerbose.METRICS:
self._print_metric_logs(show_loss, "train")
self._print_metric_logs(show_loss, "cv")
if visualize:
if visualize_setting is None:
self.visualize2d(x_test, y_test)
else:
self.visualize2d(x_test, y_test, *visualize_setting)
if self.verbose >= NNVerbose.EPOCH:
bar.update(counter // record_period + 1)
if self.verbose >= NNVerbose.ITER:
sub_bar = ProgressBar(max_value=train_repeat * record_period - 1, name="Iteration", start=False)
if do_log:
self._append_log(x_test, y_test, "test", get_loss=show_loss)
if img is not None:
cv2.waitKey(0)
cv2.destroyAllWindows()
if draw_weights:
ts = np.arange(epoch * train_repeat + 1)
for i, weight in enumerate(self._weights):
plt.figure()
for j in range(len(weight)):
plt.plot(ts, weight_trace[i][j])
plt.title("Weights toward layer {} ({})".format(i + 1, self._layers[i].name))
plt.show()
self._handle_mp4(ims, animation_properties, "NN")
return self._logs
def fit(self, x=None, y=None, lr=0.01, epoch=10, batch_size=128, train_rate=None,
verbose=0, metrics=None, record_period=100):
self.verbose = verbose
x = NNDist._transfer_x(x)
self._optimizer = Adam(lr)
self._tfx = tf.placeholder(tf.float32, shape=[None, *x.shape[1:]])
self._tfy = tf.placeholder(tf.float32, shape=[None, y.shape[1]])
if train_rate is not None:
train_rate = float(train_rate)
train_len = int(len(x) * train_rate)
shuffle_suffix = np.random.permutation(int(len(x)))
x, y = x[shuffle_suffix], y[shuffle_suffix]
x_train, y_train = x[:train_len], y[:train_len]
x_test, y_test = x[train_len:], y[train_len:]
else:
x_train = x_test = x
y_train = y_test = y
train_len = len(x_train)
batch_size = min(batch_size, train_len)
do_random_batch = train_len >= batch_size
train_repeat = int(train_len / batch_size) + 1
self._metrics = ["acc"] if metrics is None else metrics
for i, metric in enumerate(self._metrics):
if isinstance(metric, str):
self._metrics[i] = self._available_metrics[metric]
self._metric_names = [_m.__name__ for _m in self._metrics]
self._logs = {
name: [[] for _ in range(len(self._metrics) + 1)] for name in ("train", "test")
}
bar = ProgressBar(min_value=0, max_value=max(1, epoch // record_period), name="Epoch")
if self.verbose >= NNVerbose.EPOCH:
bar.start()
with self._sess.as_default() as sess:
# Define session
self._cost = self.get_rs(self._tfx, self._tfy)
self._y_pred = self.get_rs(self._tfx)
self._train_step = self._optimizer.minimize(self._cost)
sess.run(tf.global_variables_initializer())
# Train
sub_bar = ProgressBar(min_value=0, max_value=train_repeat * record_period - 1, name="Iteration")
for counter in range(epoch):
if self.verbose >= NNVerbose.EPOCH and counter % record_period == 0:
sub_bar.start()
for _i in range(train_repeat):
if do_random_batch:
batch = np.random.choice(train_len, batch_size)
x_batch, y_batch = x_train[batch], y_train[batch]
else:
x_batch, y_batch = x_train, y_train
self._train_step.run(feed_dict={self._tfx: x_batch, self._tfy: y_batch})
if self.verbose >= NNVerbose.EPOCH:
if sub_bar.update() and self.verbose >= NNVerbose.METRICS_DETAIL:
self._append_log(x_train, y_train, "train")
self._append_log(x_test, y_test, "test")
self._print_metric_logs("train")
self._print_metric_logs("test")
if self.verbose >= NNVerbose.EPOCH:
sub_bar.update()
if (counter + 1) % record_period == 0:
self._append_log(x_train, y_train, "train")
self._append_log(x_test, y_test, "test")
if self.verbose >= NNVerbose.METRICS:
self._print_metric_logs("train")
self._print_metric_logs("test")
if self.verbose >= NNVerbose.EPOCH:
bar.update(counter // record_period + 1)
sub_bar = ProgressBar(min_value=0, max_value=train_repeat * record_period - 1, name="Iteration")
def fit(self,
x=None,
y=None,
x_test=None,
y_test=None,
lr=0.01,
lb=0.01,
epoch=20,
weight_scale=1,
batch_size=256,
record_period=1,
optimizer=None,
show_loss=True,
metrics=None,
do_log=False,
verbose=None):
x, y = self._feed_data(x, y)
self._lr = lr
self._init_optimizer(optimizer)
if not self._layers:
raise BuildNetworkError(
"Please provide layers before fitting data")
if y.shape[1] != self._current_dimension:
raise BuildNetworkError(
"Output layer's shape should be {}, {} found".format(
self._current_dimension, y.shape[1]))
x_train, y_train, x_test = x, y, NNDist._transfer_x(x_test)
train_len = len(x_train)
batch_size = min(batch_size, train_len)
do_random_batch = train_len >= batch_size
train_repeat = int(train_len / batch_size) + 1
with tf.name_scope("Entry"):
self._tfx = tf.placeholder(tf.float32, shape=[None, *x.shape[1:]])
self._tfy = tf.placeholder(tf.float32, shape=[None, y.shape[1]])
if epoch <= 0:
return
self._metrics = ["acc"] if metrics is None else metrics
for i, metric in enumerate(self._metrics):
if isinstance(metric, str):
if metric not in self._available_metrics:
raise BuildNetworkError(
"Metric '{}' is not implemented".format(metric))
self._metrics[i] = self._available_metrics[metric]
self._metric_names = [_m.__name__ for _m in self._metrics]
self._logs = {
name: [[] for _ in range(len(self._metrics) + 1)]
for name in ("train", "test")
}
if verbose is not None:
self.verbose = verbose
bar = ProgressBar(min_value=0,
max_value=max(1, epoch // record_period),
name="Epoch")
if self.verbose >= NNVerbose.EPOCH and epoch > 0:
bar.start()
img = None
with self._sess.as_default() as sess:
# Session
self._y_pred = self.get_rs(self._tfx)
self._loss = self.get_rs(self._tfx,
self._tfy) + self._get_l2_loss(lb)
self._activations = self._get_activations(self._tfx)
self._init_train_step(sess)
for weight in self._tf_weights:
weight *= weight_scale
sub_bar = ProgressBar(min_value=0,
max_value=train_repeat * record_period - 1,
name="Iteration")
for counter in range(epoch):
if self.verbose >= NNVerbose.EPOCH and counter % record_period == 0:
sub_bar.start()
for i in range(train_repeat):
if do_random_batch:
batch = np.random.choice(train_len, batch_size)
x_batch, y_batch = x_train[batch], y_train[batch]
else:
x_batch, y_batch = x_train, y_train
feed_dict = {self._tfx: x_batch, self._tfy: y_batch}
self._train_step.run(feed_dict=feed_dict)
if self.verbose >= NNVerbose.DEBUG:
pass
if self.verbose >= NNVerbose.EPOCH:
if sub_bar.update(
) and self.verbose >= NNVerbose.METRICS_DETAIL:
self._append_log(x, y, "train", get_loss=show_loss)
self._append_log(x_test,
y_test,
"test",
get_loss=show_loss)
self._print_metric_logs(show_loss, "train")
self._print_metric_logs(show_loss, "test")
if self.verbose >= NNVerbose.EPOCH:
sub_bar.update()
if (counter + 1) % record_period == 0:
if do_log:
self._append_log(x, y, "train", get_loss=show_loss)
if x_test is not None:
self._append_log(x_test,
y_test,
"test",
get_loss=show_loss)
if self.verbose >= NNVerbose.METRICS:
self._print_metric_logs(show_loss, "train")
if x_test is not None:
self._print_metric_logs(show_loss, "test")
if self.verbose >= NNVerbose.EPOCH:
bar.update(counter // record_period + 1)
sub_bar = ProgressBar(
min_value=0,
max_value=train_repeat * record_period - 1,
name="Iteration")
if img is not None:
cv2.waitKey(0)
cv2.destroyAllWindows()
return self._logs
def fit(self, x=None, y=None, lr=0.01, epoch=10, batch_size=128, train_rate=None,
verbose=0, metrics=None, record_period=100):
self.verbose = verbose
self._optimizer = Adam(lr)
self._tfx = tf.placeholder(tf.float32, shape=[None, x.shape[1]])
self._tfy = tf.placeholder(tf.float32, shape=[None, y.shape[1]])
if train_rate is not None:
train_rate = float(train_rate)
train_len = int(len(x) * train_rate)
shuffle_suffix = np.random.permutation(int(len(x)))
x, y = x[shuffle_suffix], y[shuffle_suffix]
x_train, y_train = x[:train_len], y[:train_len]
x_test, y_test = x[train_len:], y[train_len:]
else:
x_train = x_test = x
y_train = y_test = y
train_len = len(x_train)
batch_size = min(batch_size, train_len)
do_random_batch = train_len >= batch_size
train_repeat = int(train_len / batch_size) + 1
self._metrics = ["acc"] if metrics is None else metrics
for i, metric in enumerate(self._metrics):
if isinstance(metric, str):
self._metrics[i] = self._available_metrics[metric]
self._metric_names = [_m.__name__ for _m in self._metrics]
self._logs = {
name: [[] for _ in range(len(self._metrics) + 1)] for name in ("train", "test")
}
bar = ProgressBar(max_value=max(1, epoch // record_period), name="Epoch", start=False)
if self.verbose >= NNVerbose.EPOCH:
bar.start()
with self._sess.as_default() as sess:
# Define session
self._cost = self.get_rs(self._tfx, self._tfy)
self._y_pred = self.get_rs(self._tfx)
self._train_step = self._optimizer.minimize(self._cost)
sess.run(tf.global_variables_initializer())
# Train
sub_bar = ProgressBar(max_value=train_repeat * record_period - 1, name="Iteration", start=False)
for counter in range(epoch):
if self.verbose >= NNVerbose.EPOCH and counter % record_period == 0:
sub_bar.start()
for _i in range(train_repeat):
if do_random_batch:
batch = np.random.choice(train_len, batch_size)
x_batch, y_batch = x_train[batch], y_train[batch]
else:
x_batch, y_batch = x_train, y_train
self._train_step.run(feed_dict={self._tfx: x_batch, self._tfy: y_batch})
if self.verbose >= NNVerbose.EPOCH:
if sub_bar.update() and self.verbose >= NNVerbose.METRICS_DETAIL:
self._append_log(x_train, y_train, "train")
self._append_log(x_test, y_test, "test")
self._print_metric_logs("train")
self._print_metric_logs("test")
if self.verbose >= NNVerbose.EPOCH:
sub_bar.update()
if (counter + 1) % record_period == 0:
self._append_log(x_train, y_train, "train")
self._append_log(x_test, y_test, "test")
if self.verbose >= NNVerbose.METRICS:
self._print_metric_logs("train")
self._print_metric_logs("test")
if self.verbose >= NNVerbose.EPOCH:
bar.update(counter // record_period + 1)
sub_bar = ProgressBar(max_value=train_repeat * record_period - 1, name="Iteration", start=False)
def fit(self,
x,
y,
lr=None,
epoch=None,
batch_size=None,
train_rate=None,
optimizer=None,
metrics=None,
record_period=None,
verbose=None):
if lr is None:
lr = self._params["lr"]
if epoch is None:
epoch = self._params["epoch"]
if optimizer is None:
optimizer = self._params["optimizer"]
if batch_size is None:
batch_size = self._params["batch_size"]
if train_rate is None:
train_rate = self._params["train_rate"]
if metrics is None:
metrics = self._params["metrics"]
if record_period is None:
record_period = self._params["record_period"]
if verbose is None:
verbose = self._params["verbose"]
self.verbose = verbose
self._init_optimizers(optimizer, lr, epoch)
layer_width = len(self._layers)
self._preview()
"""
如果train_rate不为空
并打乱数据
划分训练和测试集
"""
if train_rate is not None:
train_rate = float(train_rate)
train_len = int(len(x) * train_rate)
shuffle_suffix = np.random.permutation(len(x))
x, y = x[shuffle_suffix], y[shuffle_suffix]
x_train, y_train = x[:train_len], y[:train_len]
x_test, y_test = x[train_len:], y[train_len:]
else:
x_train = x_test = x
y_train = y_test = y
"""
y是one_hot形式，每个样本的类别通过每个数组中值最大值得下标作为类别
axis=1表示横向取最大值
a = np.array([[0,0,0,1]])
c = np.argmax(a,axis=1)
c = array([3], dtype=int64)
"""
y_train_classes = np.argmax(y_train, axis=1)
y_test_classes = np.argmax(y_test, axis=1)
train_len = len(x_train)
batch_size = min(batch_size, train_len)
do_random_batch = train_len > batch_size
train_repeat = 1 if not do_random_batch else int(train_len /
batch_size) + 1
if metrics is None:
metrics = []
self._metrics = self.get_metrics(metrics)
self._metric_names = [_m.__name__ for _m in metrics] #"acc"
self._logs = {
name: [[] for _ in range(len(metrics) + 1)]
for name in ("train", "test")
}
bar = ProgressBar(max_value=max(1, epoch // record_period),
name="Epoch",
start=False)
if self.verbose >= NNVerbose.EPOCH:
bar.start()
sub_bar = ProgressBar(max_value=train_repeat * record_period - 1,
name="Iteration",
start=False)
for counter in range(epoch):
if self.verbose >= NNVerbose.EPOCH and counter % record_period == 0:
sub_bar.start()
for _ in range(train_repeat):
if do_random_batch:
"""如果随机batch则，在train_len中选择batch_size个样本"""
batch = np.random.choice(train_len, batch_size)
x_batch, y_batch = x_train[batch], y_train[batch]
else:
x_batch, y_batch = x_train, y_train
self._w_optimizer.update()
self._b_optimizer.update()
activations = self._get_activations(x_batch)
"""
反向传播 :bp
先求出第一个局部梯度，然后后面的梯度和前一个梯度有关系 &(i) = &(i+1) * w_i^T . u'(i)
更新w,b :opt
"""
deltas = [self._layers[-1].bp_first(y_batch, activations[-1])]
for i in range(-1, -len(activations), -1): #i=-1,-2, step=-1
deltas.append(self._layers[i - 1].bp(
deltas[-1], self._weights[i], activations[i - 1])) #
for i in range(layer_width - 1, 0, -1): #i=2,1 ,step=-1
self._opt(i, activations[i - 1],
deltas[layer_width - i - 1])
#第一层对应的是输入
self._opt(
0, x_batch, deltas[-1]
) #对应的是第一个隐藏层， W_(i-1)' = v^T_(i-1) * delta_(i) delta[0]是损失层，delta[-1]是第一层
if self.verbose >= NNVerbose.EPOCH:
if sub_bar.update(
) and self.verbose >= NNVerbose.METRICS_DETAIL:
self._append_log(x_train, y_train, y_train_classes,
"train")
self._append_log(x_test, y_test, y_test_classes,
"test")
self._print_metric_logs("train")
self._print_metric_logs("test")
if self.verbose >= NNVerbose.EPOCH:
sub_bar.update()
if (counter + 1) % record_period == 0:
self._append_log(x_train, y_train, y_train_classes, "train")
self._append_log(x_test, y_test, y_test_classes, "test")
if self.verbose >= NNVerbose.METRICS:
self._print_metric_logs("train")
self._print_metric_logs("test")
if self.verbose >= NNVerbose.EPOCH:
bar.update(counter // record_period + 1)
sub_bar = ProgressBar(
max_value=train_repeat * record_period - 1,
name="Iteration",
start=False)
def fit(self,
im,
om,
generator,
cell=LSTMCell,
n_hidden=128,
n_history=0,
squeeze=None,
use_final_state=None,
activation=None,
lr=0.01,
epoch=10,
n_iter=128,
batch_size=64,
optimizer="Adam",
eps=1e-8,
verbose=1):
if squeeze:
self._squeeze = True
if use_final_state:
self._use_final_state = True
if callable(activation):
self._activation = activation
self._generator = generator
self._im, self._om = im, om
self._optimizer = OptFactory().get_optimizer_by_name(optimizer, lr)
self._define_input(im, om)
cell = cell(n_hidden)
initial_state = cell.zero_state(tf.shape(self._input)[0], tf.float32)
rnn_outputs, rnn_final_state = tf.nn.dynamic_rnn(
cell, self._input, initial_state=initial_state)
self._get_output(rnn_outputs, rnn_final_state, n_history)
loss = self._get_loss(eps)
train_step = self._optimizer.minimize(loss)
self._log["iter_err"] = []
self._log["epoch_err"] = []
self._sess.run(tf.global_variables_initializer())
bar = ProgressBar(max_value=epoch, name="Epoch", start=False)
if verbose >= 2:
bar.start()
for _ in range(epoch):
epoch_err = 0
sub_bar = ProgressBar(max_value=n_iter, name="Iter", start=False)
if verbose >= 2:
sub_bar.start()
for __ in range(n_iter):
x_batch, y_batch = self._generator.gen(batch_size)
iter_err = self._sess.run([loss, train_step], {
self._tfx: x_batch,
self._tfy: y_batch,
})[0]
self._log["iter_err"].append(iter_err)
epoch_err += iter_err
if verbose >= 2:
sub_bar.update()
self._log["epoch_err"].append(epoch_err / n_iter)
if verbose >= 1:
self._verbose()
if verbose >= 2:
bar.update()
def fit(self, x, y, lr=None, epoch=None, batch_size=None, train_rate=None,
optimizer=None, metrics=None, record_period=None, verbose=None):
if lr is None:
lr = self._params["lr"]
if epoch is None:
epoch = self._params["epoch"]
if optimizer is None:
optimizer = self._params["optimizer"]
if batch_size is None:
batch_size = self._params["batch_size"]
if train_rate is None:
train_rate = self._params["train_rate"]
if metrics is None:
metrics = self._params["metrics"]
if record_period is None:
record_period = self._params["record_period"]
if verbose is None:
verbose = self._params["verbose"]
self.verbose = verbose
self._init_optimizers(optimizer, lr, epoch)
layer_width = len(self._layers)
self._preview()
if train_rate is not None:
train_rate = float(train_rate)
train_len = int(len(x) * train_rate)
shuffle_suffix = np.random.permutation(len(x))
x, y = x[shuffle_suffix], y[shuffle_suffix]
x_train, y_train = x[:train_len], y[:train_len]
x_test, y_test = x[train_len:], y[train_len:]
else:
x_train = x_test = x
y_train = y_test = y
y_train_classes = np.argmax(y_train, axis=1)
y_test_classes = np.argmax(y_test, axis=1)
train_len = len(x_train)
batch_size = min(batch_size, train_len)
do_random_batch = train_len > batch_size
train_repeat = 1 if not do_random_batch else int(train_len / batch_size) + 1
if metrics is None:
metrics = []
self._metrics = self.get_metrics(metrics)
self._metric_names = [_m.__name__ for _m in metrics]
self._logs = {
name: [[] for _ in range(len(metrics) + 1)] for name in ("train", "test")
}
bar = ProgressBar(max_value=max(1, epoch // record_period), name="Epoch", start=False)
if self.verbose >= NNVerbose.EPOCH:
bar.start()
sub_bar = ProgressBar(max_value=train_repeat * record_period - 1, name="Iteration", start=False)
for counter in range(epoch):
if self.verbose >= NNVerbose.EPOCH and counter % record_period == 0:
sub_bar.start()
for _ in range(train_repeat):
if do_random_batch:
batch = np.random.choice(train_len, batch_size)
x_batch, y_batch = x_train[batch], y_train[batch]
else:
x_batch, y_batch = x_train, y_train
self._w_optimizer.update()
self._b_optimizer.update()
activations = self._get_activations(x_batch)
deltas = [self._layers[-1].bp_first(y_batch, activations[-1])]
for i in range(-1, -len(activations), -1):
deltas.append(
self._layers[i - 1].bp(activations[i - 1], self._weights[i], deltas[-1])
)
for i in range(layer_width - 1, 0, -1):
self._opt(i, activations[i - 1], deltas[layer_width - i - 1])
self._opt(0, x_batch, deltas[-1])
if self.verbose >= NNVerbose.EPOCH:
if sub_bar.update() and self.verbose >= NNVerbose.METRICS_DETAIL:
self._append_log(x_train, y_train, y_train_classes, "train")
self._append_log(x_test, y_test, y_test_classes, "test")
self._print_metric_logs("train")
self._print_metric_logs("test")
if self.verbose >= NNVerbose.EPOCH:
sub_bar.update()
if (counter + 1) % record_period == 0:
self._append_log(x_train, y_train, y_train_classes, "train")
self._append_log(x_test, y_test, y_test_classes, "test")
if self.verbose >= NNVerbose.METRICS:
self._print_metric_logs("train")
self._print_metric_logs("test")
if self.verbose >= NNVerbose.EPOCH:
bar.update(counter // record_period + 1)
sub_bar = ProgressBar(max_value=train_repeat * record_period - 1, name="Iteration", start=False)
def fit(self,
x,
y,
x_test,
y_test,
n_epoch=None,
n_batch=None,
print_settings=True):
if not self.settings_inited:
self.init_all_settings()
if n_epoch is not None:
self.n_epoch = n_epoch
if n_batch is not None:
self.n_batch = n_batch
x, y, x_test, y_test = self.prepare_data(x, y, x_test, y_test)
if not self.model_built:
self.build_model(x, y, x_test, y_test, print_settings)
count = 0
train_losses, test_losses = [], []
with self._sess.as_default() as sess:
# Prepare
i = 0
train_writer, test_writer, train_merge_op, test_merge_op = self._prepare_tensorboard_verbose(
sess)
bar = ProgressBar(max_value=self.n_epoch, name="Main")
train_info = [train_merge_op, train_losses, train_writer]
test_info = [test_merge_op, test_losses, test_writer]
self._calculate_loss(train_losses, test_losses)
train_metric, test_metric = self._get_metrics(x, y, x_test, y_test)
if self.tensorboard_verbose > 0:
self._do_tensorboard_verbose(count, train_info, test_info,
train_metric, test_metric)
# Train
while i < self.n_epoch:
for local_dict in self.gen_dicts(self.train_data, count=count):
count += 1
self._sess.run(self._train_step, local_dict)
if self.snapshot_step > 0 and count % self.snapshot_step == 0:
if self.tensorboard_verbose > 0:
train_metric, test_metric = self._get_metrics(
x, y, x_test, y_test)
self._do_tensorboard_verbose(
count, train_info, test_info, train_metric,
test_metric)
i += 1
if self.tensorboard_verbose > 0:
if train_metric is None:
y_pred, y_test_pred = self._calculate_loss(
train_losses, test_losses, return_pred=True)
train_metric = self.metric(y, y_pred)
test_metric = self.metric(
y_test,
y_test_pred) if y_test is not None else None
else:
self._calculate_loss(train_losses, test_losses)
self._do_tensorboard_verbose(count, train_info, test_info,
train_metric, test_metric)
else:
self._calculate_loss(train_losses, test_losses)
if bar is not None:
bar.update()
return train_losses, test_losses
def fit(self,
x,
y,
lr=None,
epoch=None,
batch_size=None,
train_rate=None,
optimizer=None,
metrics=None,
record_period=None,
verbose=None,
preview=None):
if lr is None:
lr = self._params["lr"]
if epoch is None:
epoch = self._params["epoch"]
if optimizer is None:
optimizer = self._params["optimizer"]
if batch_size is None:
batch_size = self._params["batch_size"]
if train_rate is None:
train_rate = self._params["train_rate"]
if metrics is None:
metrics = self._params["metrics"]
if record_period is None:
record_period = self._params["record_period"]
if verbose is None:
verbose = self._params["verbose"]
if preview is None:
preview = self._params["preview"]
x = NN._transfer_x(x)
self.verbose = verbose
self._optimizers = OptFactory().get_optimizer_by_name(optimizer, lr)
#None=batch_size的占位符, *很有技巧
"""
a=np.array([[12,21],[123,2]])
def get(a):
print(*a.shape)
print(a.shape)
get(a)
2 2
(2, 2)
"""
self._tfx = tf.placeholder(tf.float32, shape=[None, *x.shape[1:]])
self._tfy = tf.placeholder(tf.float32, shape=[None, y.shape[1]])
if train_rate is not None:
train_rate = float(train_rate)
train_len = int(len(x) * train_rate)
shuffle_shuffix = np.random.permutation(int(len(x)))
x, y = x[shuffle_shuffix], y[shuffle_shuffix]
x_train, y_train = x[:train_len], y[:train_len]
x_test, y_test = x[train_len:], y[train_len:]
else:
x_train = x_test = x
y_train = y_test = y
y_train_classes = np.argmax(y_train, axis=1)
y_test_classes = np.argmax(y_test, axis=1)
if metrics is None:
metrics = []
self._metrics = self.get_metrics(metrics)
self._metric_names = [_m.__name__ for _m in metrics]
self._logs = {
name: [[] for _ in range(len(metrics) + 1)]
for name in ("Train", "Test")
}
bar = ProgressBar(max_value=max(1, epoch // record_period),
name="Epoch",
start=False)
if self.verbose >= NNVerbose.EPOCH:
bar.start()
if preview:
self._preview()
args = ((x_train, y_train, y_train_classes, x_test, y_test,
y_test_classes, self.verbose >= NNVerbose.METRICS_DETAIL),
(None, None, x_train, y_train, y_train_classes, x_test, y_test,
y_test_classes, self.verbose >= NNVerbose.METRICS))
train_repeat = self._get_train_repeat(x, batch_size)
with self._sess.as_default() as sess:
self._y_pred = self._get_rs(self._tfx)
self._inner_y = self._get_rs(self._tfx, predict=False)
self._loss = self._layers[-1].calculate(self._tfy, self._inner_y)
self._train_step = self._optimizers.minimize(self._loss)
sess.run(tf.global_variables_initializer())
for counter in range(epoch):
if self.verbose >= NNVerbose.ITER and counter % record_period == 0:
sub_bar = ProgressBar(
max_value=train_repeat * record_period - 1,
name="Iteration")
else:
sub_bar = None
self._batch_training(x_train, y_train, batch_size,
train_repeat, self._loss,
self._train_step, sub_bar, *args[0])
if (counter + 1) % record_period == 0:
self._batch_work(*args[1])
if self.verbose >= NNVerbose.EPOCH:
bar.update(counter // record_period + 1)
def fit(self,
x, y, x_test=None, y_test=None,
batch_size=128, record_period=1, train_only=False,
optimizer="Adam", lr=0.001, lb=0.001, epoch=20, weight_scale=1, apply_bias=True,
show_loss=True, metrics=None, do_log=True, verbose=None,
visualize=False, visualize_setting=None,
draw_weights=False, animation_params=None):
self._lr, self._epoch = lr, epoch
for weight in self._weights:
if weight is not None:
weight.data *= weight_scale
self._model_parameters = [w for w in self._weights if w is not None]
if apply_bias:
self._model_parameters += self._bias
self._optimizer = optimizer
self._init_optimizer()
assert isinstance(self._optimizer, Optimizer)
print()
print("=" * 30)
print("Optimizers")
print("-" * 30)
print(self._optimizer)
print("-" * 30)
if not self._layers:
raise BuildNetworkError("Please provide layers before fitting data")
if y.shape[1] != self._current_dimension:
raise BuildNetworkError("Output layer's shape should be {}, {} found".format(
self._current_dimension, y.shape[1]))
x, y = self._arr_to_variable(False, x, y)
if x_test is not None and y_test is not None:
x_test, y_test = self._arr_to_variable(False, x_test, y_test)
(x_train, x_test), (y_train, y_test) = self._split_data(
x, y, x_test, y_test, train_only)
train_len = len(x_train)
batch_size = min(batch_size, train_len)
do_random_batch = train_len > batch_size
train_repeat = 1 if not do_random_batch else int(train_len / batch_size) + 1
self._regularization_param = 1 - lb * lr / batch_size
self._get_min_max(x_train, y_train)
self._metrics = ["acc"] if metrics is None else metrics
for i, metric in enumerate(self._metrics):
if isinstance(metric, str):
if metric not in self._available_metrics:
raise BuildNetworkError("Metric '{}' is not implemented".format(metric))
self._metrics[i] = self._available_metrics[metric]
self._metric_names = [_m.__name__ for _m in self._metrics]
self._logs = {
name: [[] for _ in range(len(self._metrics) + 1)] for name in ("train", "cv", "test")
}
if verbose is not None:
self.verbose = verbose
self._apply_bias = apply_bias
bar = ProgressBar(max_value=max(1, epoch // record_period), name="Epoch", start=False)
if self.verbose >= NNVerbose.EPOCH:
bar.start()
img, ims = None, []
if draw_weights:
weight_trace = [[[org] for org in weight] for weight in self._weights]
else:
weight_trace = []
loss_function = self._layers[-1].calculate
args = (
x_train, y_train, x_test, y_test,
draw_weights, weight_trace, show_loss
)
*animation_properties, animation_params = self._get_animation_params(animation_params)
sub_bar = ProgressBar(max_value=train_repeat * record_period - 1, name="Iteration", start=False)
for counter in range(epoch):
self._optimizer.update()
if self.verbose >= NNVerbose.ITER and counter % record_period == 0:
sub_bar.start()
self.batch_training(
x_train, y_train, batch_size, train_repeat, loss_function, sub_bar, *args
)
if self.verbose >= NNVerbose.ITER:
sub_bar.update()
self._handle_animation(
counter, x, y, ims, animation_params, *animation_properties,
img=self._draw_2d_network(**animation_params), name="Neural Network"
)
if do_log:
self._append_log(x, y, "train", get_loss=show_loss)
self._append_log(x_test, y_test, "cv", get_loss=show_loss)
if (counter + 1) % record_period == 0:
if do_log and self.verbose >= NNVerbose.METRICS:
self._print_metric_logs(show_loss, "train")
self._print_metric_logs(show_loss, "cv")
if visualize:
if visualize_setting is None:
self.visualize2d(x_test, y_test)
else:
self.visualize2d(x_test, y_test, *visualize_setting)
if self.verbose >= NNVerbose.EPOCH:
bar.update(counter // record_period + 1)
if self.verbose >= NNVerbose.ITER:
sub_bar = ProgressBar(max_value=train_repeat * record_period - 1, name="Iteration", start=False)
if do_log:
self._append_log(x_test, y_test, "test", get_loss=show_loss)
if img is not None:
cv2.waitKey(0)
cv2.destroyAllWindows()
if draw_weights:
ts = np.arange(epoch * train_repeat + 1)
for i, weight in enumerate(self._weights):
plt.figure()
for j in range(len(weight)):
plt.plot(ts, weight_trace[i][j])
plt.title("Weights toward layer {} ({})".format(i + 1, self._layers[i].name))
plt.show()
self._handle_mp4(ims, animation_properties, "NN")
return self._logs
def fit(self,
x,
y,
sample_weight=None,
kernel=None,
epoch=None,
x_test=None,
y_test=None,
metrics=None,
**kwargs):
if sample_weight is None:
sample_weight = self._params["sw"]
if kernel is None:
kernel = self._params["kernel"]
if epoch is None:
epoch = self._params["epoch"]
if x_test is None:
x_test = self._params["x_test"]
if y_test is None:
y_test = self._params["y_test"]
if metrics is None:
metrics = self._params["metrics"]
self._x, self._y = np.atleast_2d(x), np.asarray(y)
if kernel == "poly":
_p = kwargs.get("p", self._params["p"])
self._kernel_name = "Polynomial"
self._kernel_param = "degree = {}".format(_p)
self._kernel = lambda _x, _y: KernelBase._poly(_x, _y, _p)
elif kernel == "rbf":
_gamma = kwargs.get("gamma", 1 / self._x.shape[1])
self._kernel_name = "RBF"
self._kernel_param = r"$\gamma = {:8.6}$".format(_gamma)
self._kernel = lambda _x, _y: KernelBase._rbf(_x, _y, _gamma)
else:
raise NotImplementedError(
"Kernel '{}' has not defined".format(kernel))
if sample_weight is None:
sample_weight = np.ones(len(y))
else:
sample_weight = np.asarray(sample_weight) * len(y)
self._alpha, self._w, self._prediction_cache = (np.zeros(len(x)),
np.zeros(len(x)),
np.zeros(len(x)))
self._gram = self._kernel(self._x, self._x)
self._b = 0
self._prepare(sample_weight, **kwargs)
_fit_args, _logs = [], []
for _name, _arg in zip(self._fit_args_names, self._fit_args):
if _name in kwargs:
_arg = kwargs[_name]
_fit_args.append(_arg)
if self._do_log:
if metrics is not None:
self.get_metrics(metrics)
_test_gram = None
if x_test is not None and y_test is not None:
_xv, _yv = np.atleast_2d(x_test), np.asarray(y_test)
_test_gram = self._kernel(_xv, self._x)
else:
_xv, _yv = self._x, self._y
else:
_yv = _test_gram = None
bar = ProgressBar(max_value=epoch, name=str(self))
bar.start()
for _ in range(epoch):
if self._fit(sample_weight, *_fit_args):
bar.update(epoch)
break
if self._do_log and metrics is not None:
_local_logs = []
for metric in metrics:
if _test_gram is None:
_local_logs.append(
metric(self._y, np.sign(self._prediction_cache)))
else:
_local_logs.append(
metric(
_yv,
self.predict(_test_gram, gram_provided=True)))
_logs.append(_local_logs)
bar.update()
return _logs
def fit(self,
im,
om,
generator,
cell=None,
provide_sequence_length=None,
squeeze=None,
use_sparse_labels=None,
embedding_size=None,
use_final_state=None,
n_hidden=None,
n_history=None,
activation=None,
lr=None,
epoch=None,
n_iter=None,
batch_size=None,
optimizer=None,
eps=None,
verbose=None):
if cell is None:
cell = self._params["cell"]
if provide_sequence_length is None:
provide_sequence_length = self._params["provide_sequence_length"]
if n_hidden is None:
n_hidden = self._params["n_hidden"]
if n_history is None:
n_history = self._params["n_history"]
if squeeze:
self._squeeze = True
if use_sparse_labels:
self._use_sparse_labels = True
if self._squeeze and n_history == 0:
n_history = 1
if embedding_size:
self._embedding_size = embedding_size
if use_final_state:
self._use_final_state = True
if activation is None:
activation = self._params["activation"]
if lr is None:
lr = self._params["lr"]
if epoch is None:
epoch = self._params["epoch"]
if n_iter is None:
n_iter = self._params["n_iter"]
if optimizer is None:
optimizer = self._params["optimizer"]
if batch_size is None:
batch_size = self._params["batch_size"]
if eps is None:
eps = self._params["eps"]
if verbose is None:
verbose = self._params["verbose"]
self._generator = generator
self._im, self._om, self._activation = im, om, activation
self._optimizer = OptFactory().get_optimizer_by_name(optimizer, lr)
self._define_input(im, om)
self._cell = cell(n_hidden)
self._prepare_for_dynamic_rnn(provide_sequence_length)
rnn_outputs, rnn_final_state = tf.nn.dynamic_rnn(
self._cell,
self._input,
return_all_states=True,
sequence_length=self._sequence_lengths,
initial_state=self._initial_state)
self._get_output(rnn_outputs, rnn_final_state, n_history)
loss = self._get_loss(eps)
train_step = self._optimizer.minimize(loss)
self._log["iter_err"] = []
self._log["epoch_err"] = []
self._sess.run(tf.global_variables_initializer())
bar = ProgressBar(max_value=epoch, name="Epoch", start=False)
if verbose >= 2:
bar.start()
for _ in range(epoch):
epoch_err = 0
sub_bar = ProgressBar(max_value=n_iter, name="Iter", start=False)
if verbose >= 2:
sub_bar.start()
for __ in range(n_iter):
if provide_sequence_length:
x_batch, y_batch, sequence_length = self._generator.gen(
batch_size)
feed_dict = {
self._tfx: x_batch,
self._tfy: y_batch,
self._sequence_lengths: sequence_length
}
else:
x_batch, y_batch = self._generator.gen(batch_size)
feed_dict = {self._tfx: x_batch, self._tfy: y_batch}
iter_err = self._sess.run([loss, train_step], feed_dict)[0]
self._log["iter_err"].append(iter_err)
epoch_err += iter_err
if verbose >= 2:
sub_bar.update()
self._log["epoch_err"].append(epoch_err / n_iter)
if verbose >= 1:
self._verbose()
if verbose >= 2:
bar.update()
def _extract(self, verbose):
features = []
with tf.Session() as sess:
if self._extractor == "v3":
_output = "pool_3:0"
elif self._extractor == "ResNet-v2":
_output = "InceptionResnetV2/Logits/Flatten/Reshape:0"
elif self._extractor == "cnn":
_output = "final_result/Reshape:0"
else:
_output = "OutputFlow/Reshape:0"
flattened_tensor = sess.graph.get_tensor_by_name(_output)
if self._extractor == "v3":
_entry = "DecodeJpeg/contents:0"
elif self._extractor == "ResNet-v2":
_entry = "Placeholder:0"
else:
_entry = "Entry/Placeholder:0"
pop_lst = []
if "cnn" in self._extractor or "ResNet" in self._extractor and self._mat_dir is not None:
features = np.load(self._mat_dir)
else:
def process(img_path):
img_data = gfile.FastGFile(img_path, "rb").read()
feature = sess.run(flattened_tensor, {
_entry: img_data
})
features.append(np.squeeze(feature))
for i, image_path in enumerate(self._image_paths):
if not os.path.isfile(image_path):
continue
if "v3" in self._extractor:
if verbose:
print("Processing {}...".format(image_path))
try:
process(image_path)
except Exception as err:
if verbose:
print(err)
name, extension = os.path.splitext(image_path)
base = os.path.basename(image_path)
if extension.lower() in (".jpg", ".jpeg"):
new_name = name[:image_path.rfind(base)] + "{:06d}{}".format(i, extension)
print("Renaming {} to {}...".format(image_path, new_name))
os.rename(image_path, new_name)
process(new_name)
else:
new_name_base = name[:image_path.rfind(base)] + "{:06d}".format(i)
new_name = new_name_base + ".jpg"
print("Transforming {} to {}...".format(image_path, new_name))
try:
if imghdr.what(image_path) is None:
raise ValueError("{} is not an image".format(image_path))
os.rename(image_path, new_name_base + extension)
cv2.imwrite(new_name, cv2.imread(new_name_base + extension))
os.remove(new_name_base + extension)
process(new_name)
except Exception as err:
print(err)
print("Moving {} to '_err' folder...".format(image_path))
if not os.path.isdir("_err"):
os.makedirs("_err")
shutil.move(image_path, os.path.join("_err", os.path.basename(image_path)))
pop_lst.append(i)
else:
if verbose:
print("Reading {}...".format(image_path))
image_data = cv2.imread(image_path)
if self._extractor == "ResNet-v2":
features.append(cv2.resize(image_data, (299, 299)))
else:
features.append(cv2.resize(image_data, (64, 64)))
if "v3" not in self._extractor:
features = np.array(features)
print("Extracting features...")
rs = []
batch_size = math.floor(1e6 / np.prod(features.shape[1:]))
epoch = int(math.ceil(len(features) / batch_size))
bar = ProgressBar(max_value=epoch, name="Extract")
for i in range(epoch):
if i == epoch - 1:
rs.append(sess.run(flattened_tensor, {
_entry: features[i*batch_size:]
}))
else:
rs.append(sess.run(flattened_tensor, {
_entry: features[i*batch_size:(i+1)*batch_size]
}))
bar.update()
return np.vstack(rs).astype(np.float32)
if pop_lst:
labels = []
pop_cursor, pop_idx = 0, pop_lst[0]
for i, label in enumerate(self._labels):
if i == pop_idx:
pop_cursor += 1
if pop_cursor < len(pop_lst):
pop_idx = pop_lst[pop_cursor]
else:
pop_idx = -1
continue
labels.append(label)
labels = np.array(labels, dtype=np.float32)
elif self._labels is None:
labels = None
else:
labels = np.array(self._labels, dtype=np.float32)
return np.array(features, dtype=np.float32), labels
def fit(self,
x=None,
y=None,
x_test=None,
y_test=None,
batch_size=128,
record_period=1,
train_only=False,
optimizer=None,
w_optimizer=None,
b_optimizer=None,
lr=0.001,
lb=0.001,
epoch=20,
weight_scale=1,
apply_bias=True,
show_loss=True,
metrics=None,
do_log=True,
verbose=None,
visualize=False,
visualize_setting=None,
draw_weights=False,
animation_params=None):
self._lr, self._epoch = lr, epoch
for weight in self._weights:
weight *= weight_scale
if not self._w_optimizer or not self._b_optimizer:
if not self._optimizer_name:
if optimizer is None:
optimizer = "Adam"
self._w_optimizer = optimizer if w_optimizer is None else w_optimizer
self._b_optimizer = optimizer if b_optimizer is None else b_optimizer
else:
if not self._w_optimizer:
self._w_optimizer = self._optimizer_name
if not self._b_optimizer:
self._b_optimizer = self._optimizer_name
self._init_optimizer()
assert isinstance(self._w_optimizer, Optimizer) and isinstance(
self._b_optimizer, Optimizer)
print()
print("=" * 30)
print("Optimizers")
print("-" * 30)
print("w: {}\nb: {}".format(self._w_optimizer, self._b_optimizer))
print("-" * 30)
if not self._layers:
raise BuildNetworkError(
"Please provide layers before fitting data")
if y.shape[1] != self._current_dimension:
raise BuildNetworkError(
"Output layer's shape should be {}, {} found".format(
self._current_dimension, y.shape[1]))
(x_train, x_test), (y_train,
y_test) = self._split_data(x, y, x_test, y_test,
train_only)
train_len = len(x_train)
batch_size = min(batch_size, train_len)
do_random_batch = train_len > batch_size
train_repeat = 1 if not do_random_batch else int(train_len /
batch_size) + 1
self._regularization_param = 1 - lb * lr / batch_size
self._get_min_max(x_train, y_train)
self._metrics = ["acc"] if metrics is None else metrics
for i, metric in enumerate(self._metrics):
if isinstance(metric, str):
if metric not in self._available_metrics:
raise BuildNetworkError(
"Metric '{}' is not implemented".format(metric))
self._metrics[i] = self._available_metrics[metric]
self._metric_names = [_m.__name__ for _m in self._metrics]
self._logs = {
name: [[] for _ in range(len(self._metrics) + 1)]
for name in ("train", "cv", "test")
}
if verbose is not None:
self.verbose = verbose
layer_width = len(self._layers)
self._apply_bias = apply_bias
bar = ProgressBar(max_value=max(1, epoch // record_period),
name="Epoch",
start=False)
if self.verbose >= NNVerbose.EPOCH:
bar.start()
img, ims = None, []
if draw_weights:
weight_trace = [[[org] for org in weight]
for weight in self._weights]
else:
weight_trace = []
*animation_properties, animation_params = self._get_animation_params(
animation_params)
sub_bar = ProgressBar(max_value=train_repeat * record_period - 1,
name="Iteration",
start=False)
for counter in range(epoch):
self._w_optimizer.update()
self._b_optimizer.update()
if self.verbose >= NNVerbose.ITER and counter % record_period == 0:
sub_bar.start()
for _ in range(train_repeat):
if do_random_batch:
batch = np.random.choice(train_len, batch_size)
x_batch, y_batch = x_train[batch], y_train[batch]
else:
x_batch, y_batch = x_train, y_train
activations = self._get_activations(x_batch)
deltas = [self._layers[-1].bp_first(y_batch, activations[-1])]
for i in range(-1, -len(activations), -1):
deltas.append(self._layers[i - 1].bp(
activations[i - 1], self._weights[i], deltas[-1]))
for i in range(layer_width - 1, 0, -1):
if not isinstance(self._layers[i], SubLayer):
self._opt(i, activations[i - 1],
deltas[layer_width - i - 1])
self._opt(0, x_batch, deltas[-1])
if draw_weights:
for i, weight in enumerate(self._weights):
for j, new_weight in enumerate(weight.copy()):
weight_trace[i][j].append(new_weight)
if self.verbose >= NNVerbose.DEBUG:
pass
if self.verbose >= NNVerbose.ITER:
if sub_bar.update(
) and self.verbose >= NNVerbose.METRICS_DETAIL:
self._append_log(x, y, "train", get_loss=show_loss)
self._append_log(x_test,
y_test,
"cv",
get_loss=show_loss)
self._print_metric_logs(show_loss, "train")
self._print_metric_logs(show_loss, "cv")
if self.verbose >= NNVerbose.ITER:
sub_bar.update()
self._handle_animation(
counter,
x,
y,
ims,
animation_params,
*animation_properties,
img=self._draw_2d_network(**animation_params),
name="Neural Network")
if do_log:
self._append_log(x, y, "train", get_loss=show_loss)
self._append_log(x_test, y_test, "cv", get_loss=show_loss)
if (counter + 1) % record_period == 0:
if do_log and self.verbose >= NNVerbose.METRICS:
self._print_metric_logs(show_loss, "train")
self._print_metric_logs(show_loss, "cv")
if visualize:
if visualize_setting is None:
self.visualize2d(x_test, y_test)
else:
self.visualize2d(x_test, y_test, *visualize_setting)
if self.verbose >= NNVerbose.EPOCH:
bar.update(counter // record_period + 1)
if self.verbose >= NNVerbose.ITER:
sub_bar = ProgressBar(
max_value=train_repeat * record_period - 1,
name="Iteration",
start=False)
if do_log:
self._append_log(x_test, y_test, "test", get_loss=show_loss)
if img is not None:
cv2.waitKey(0)
cv2.destroyAllWindows()
if draw_weights:
ts = np.arange(epoch * train_repeat + 1)
for i, weight in enumerate(self._weights):
plt.figure()
for j in range(len(weight)):
plt.plot(ts, weight_trace[i][j])
plt.title("Weights toward layer {} ({})".format(
i + 1, self._layers[i].name))
plt.show()
self._handle_mp4(ims, animation_properties, "NN")
return self._logs
def _extract(self, verbose):
features = []
with tf.Session() as sess:
if self._extractor == "v3":
_output = "pool_3:0"
elif self._extractor == "ResNet-v2":
_output = "InceptionResnetV2/Logits/Flatten/Reshape:0"
elif self._extractor == "cnn":
_output = "final_result/Reshape:0"
else:
_output = "OutputFlow/Reshape:0"
flattened_tensor = sess.graph.get_tensor_by_name(_output)
if self._extractor == "v3":
_entry = "DecodeJpeg/contents:0"
elif self._extractor == "ResNet-v2":
_entry = "Placeholder:0"
else:
_entry = "Entry/Placeholder:0"
if "cnn" in self._extractor or "ResNet" in self._extractor and self._mat_dir is not None:
features = np.load(self._mat_dir)
else:
for i, image_path in enumerate(self._image_paths):
if not os.path.isfile(image_path):
continue
if "v3" in self._extractor:
if verbose:
print("Processing {}...".format(image_path))
try:
image_data = gfile.FastGFile(image_path,
"rb").read()
feature = sess.run(flattened_tensor,
{_entry: image_data})
features.append(np.squeeze(feature))
except Exception as err:
if verbose:
print(err)
print("Moving {} to '_err' folder...".format(
image_path))
if not os.path.isdir("_err"):
os.makedirs("_err")
shutil.move(
image_path,
os.path.join("_err",
os.path.basename(image_path)))
else:
if verbose:
print("Reading {}...".format(image_path))
image_data = cv2.imread(image_path)
if self._extractor == "ResNet-v2":
features.append(cv2.resize(image_data, (299, 299)))
else:
features.append(cv2.resize(image_data, (64, 64)))
if "v3" not in self._extractor:
features = np.array(features)
print("Extracting features...")
rs = []
batch_size = math.floor(1e6 / np.prod(features.shape[1:]))
epoch = math.ceil(len(features) / batch_size)
bar = ProgressBar(max_value=epoch, name="Extract")
bar.start()
for i in range(epoch):
if i == epoch - 1:
rs.append(
sess.run(flattened_tensor,
{_entry: features[i * batch_size:]}))
else:
rs.append(
sess.run(
flattened_tensor, {
_entry:
features[i * batch_size:(i + 1) *
batch_size]
}))
bar.update()
return np.vstack(rs).astype(np.float32)
return np.array(features, dtype=np.float32)
def test(test_sets, prob_lists):
acc_lst = []
for i in range(10):
_prob_lst = prob_lists[i]
x_test, y_test = test_sets[i]
y_pred = np.array([pick_best(sentence, _prob_lst) for sentence in x_test])
y_test = np.array(y_test)
acc_lst.append(100 * np.sum(y_pred == y_test) / len(y_pred))
return acc_lst
if __name__ == '__main__':
_rs, epoch = [], 10
bar = ProgressBar(max_value=epoch, name="_NB")
for _ in range(epoch):
_rs.append(test(*train()))
bar.update()
_rs = np.array(_rs).T
# x_base = np.arange(len(_rs[0])) + 1
# plt.figure()
# for _acc_lst in _rs:
# plt.plot(x_base, _acc_lst)
# plt.plot(x_base, np.average(_rs, axis=0), linewidth=4, label="Average")
# plt.xlim(1, epoch)
# plt.ylim(np.min(_rs), np.max(_rs)+2)
# plt.legend(loc="lower right")
# plt.show()
plt.figure()
plt.boxplot(_rs.T, vert=False, showmeans=True)
plt.show()
_rs = np.array(_rs).ravel()
print("Acc Mean : {:8.6}".format(np.average(_rs)))
def fit(self,
x,
y,
x_test=None,
y_test=None,
batch_size=128,
record_period=1,
train_only=False,
optimizer="Adam",
lr=0.001,
lb=0.001,
epoch=20,
weight_scale=1,
apply_bias=True,
show_loss=True,
metrics=None,
do_log=True,
verbose=None,
visualize=False,
visualize_setting=None,
draw_weights=False,
animation_params=None):
self._lr, self._epoch = lr, epoch
for weight in self._weights:
weight.data *= weight_scale
self._model_parameters = self._weights
if apply_bias:
self._model_parameters += self._bias
self._optimizer = optimizer
self._init_optimizer()
assert isinstance(self._optimizer, Optimizer)
print()
print("=" * 30)
print("Optimizers")
print("-" * 30)
print(self._optimizer)
print("-" * 30)
if not self._layers:
raise BuildNetworkError(
"Please provide layers before fitting data")
if y.shape[1] != self._current_dimension:
raise BuildNetworkError(
"Output layer's shape should be {}, {} found".format(
self._current_dimension, y.shape[1]))
x, y = self._arr_to_variable(False, x, y)
if x_test is not None and y_test is not None:
x_test, y_test = self._arr_to_variable(False, x_test, y_test)
(x_train, x_test), (y_train,
y_test) = self._split_data(x, y, x_test, y_test,
train_only)
train_len = len(x_train)
batch_size = min(batch_size, train_len)
do_random_batch = train_len > batch_size
train_repeat = 1 if not do_random_batch else int(train_len /
batch_size) + 1
self._regularization_param = 1 - lb * lr / batch_size
self._get_min_max(x_train, y_train)
self._metrics = ["acc"] if metrics is None else metrics
for i, metric in enumerate(self._metrics):
if isinstance(metric, str):
if metric not in self._available_metrics:
raise BuildNetworkError(
"Metric '{}' is not implemented".format(metric))
self._metrics[i] = self._available_metrics[metric]
self._metric_names = [_m.__name__ for _m in self._metrics]
self._logs = {
name: [[] for _ in range(len(self._metrics) + 1)]
for name in ("train", "cv", "test")
}
if verbose is not None:
self.verbose = verbose
self._apply_bias = apply_bias
bar = ProgressBar(max_value=max(1, epoch // record_period),
name="Epoch",
start=False)
if self.verbose >= NNVerbose.EPOCH:
bar.start()
img, ims = None, []
if draw_weights:
weight_trace = [[[org] for org in weight]
for weight in self._weights]
else:
weight_trace = []
loss_function = self._layers[-1].calculate
args = (x_train, y_train, x_test, y_test, draw_weights, weight_trace,
show_loss)
*animation_properties, animation_params = self._get_animation_params(
animation_params)
sub_bar = ProgressBar(max_value=train_repeat * record_period - 1,
name="Iteration",
start=False)
for counter in range(epoch):
self._optimizer.update()
if self.verbose >= NNVerbose.ITER and counter % record_period == 0:
sub_bar.start()
self.batch_training(x_train, y_train, batch_size, train_repeat,
loss_function, sub_bar, *args)
if self.verbose >= NNVerbose.ITER:
sub_bar.update()
self._handle_animation(
counter,
x,
y,
ims,
animation_params,
*animation_properties,
img=self._draw_2d_network(**animation_params),
name="Neural Network")
if do_log:
self._append_log(x, y, "train", get_loss=show_loss)
self._append_log(x_test, y_test, "cv", get_loss=show_loss)
if (counter + 1) % record_period == 0:
if do_log and self.verbose >= NNVerbose.METRICS:
self._print_metric_logs(show_loss, "train")
self._print_metric_logs(show_loss, "cv")
if visualize:
if visualize_setting is None:
self.visualize2d(x_test, y_test)
else:
self.visualize2d(x_test, y_test, *visualize_setting)
if self.verbose >= NNVerbose.EPOCH:
bar.update(counter // record_period + 1)
if self.verbose >= NNVerbose.ITER:
sub_bar = ProgressBar(
max_value=train_repeat * record_period - 1,
name="Iteration",
start=False)
if do_log:
self._append_log(x_test, y_test, "test", get_loss=show_loss)
if img is not None:
cv2.waitKey(0)
cv2.destroyAllWindows()
if draw_weights:
ts = np.arange(epoch * train_repeat + 1)
for i, weight in enumerate(self._weights):
plt.figure()
for j in range(len(weight)):
plt.plot(ts, weight_trace[i][j])
plt.title("Weights toward layer {} ({})".format(
i + 1, self._layers[i].name))
plt.show()
self._handle_mp4(ims, animation_properties, "NN")
return self._logs
def fit(self,
x, y, x_test=None, y_test=None,
lr=0.001, lb=0.001, epoch=10, weight_scale=1,
batch_size=128, record_period=1, train_only=False, optimizer=None,
show_loss=True, metrics=None, do_log=False, verbose=None,
tensorboard_verbose=0, animation_params=None):
x, y = self._feed_data(x, y)
self._lr = lr
self._init_optimizer(optimizer)
print("Optimizer: ", self._optimizer.name)
print("-" * 30)
if not self._layers:
raise BuildNetworkError("Please provide layers before fitting data")
if y.shape[1] != self._current_dimension:
raise BuildNetworkError("Output layer's shape should be {}, {} found".format(
self._current_dimension, y.shape[1]))
(x_train, x_test), (y_train, y_test) = self.split_data(x, y, x_test, y_test, train_only)
train_len, test_len = len(x_train), len(x_test)
batch_size = min(batch_size, train_len)
do_random_batch = train_len > batch_size
train_repeat = 1 if not do_random_batch else int(train_len / batch_size) + 1
with tf.name_scope("Entry"):
self._tfx = tf.placeholder(tf.float32, shape=[None, *x.shape[1:]])
self._tfy = tf.placeholder(tf.float32, shape=[None, y.shape[1]])
if epoch <= 0:
return
self._metrics = ["acc"] if metrics is None else metrics
for i, metric in enumerate(self._metrics):
if isinstance(metric, str):
if metric not in self._available_metrics:
raise BuildNetworkError("Metric '{}' is not implemented".format(metric))
self._metrics[i] = self._available_metrics[metric]
self._metric_names = [_m.__name__ for _m in self._metrics]
self._logs = {
name: [[] for _ in range(len(self._metrics) + 1)] for name in ("Train", "Test")
}
if verbose is not None:
self.verbose = verbose
bar = ProgressBar(max_value=max(1, epoch // record_period), name="Epoch", start=False)
if self.verbose >= NNVerbose.EPOCH:
bar.start()
img = None
*animation_properties, animation_params = self._get_animation_params(animation_params)
with self._sess.as_default() as sess:
with tf.name_scope("ActivationFlow"):
self._activations = self._get_activations(self._tfx)
self._y_pred = self._activations[-1]
l2_losses = self._get_l2_losses(lb) # type: list
self._loss = self._layers[-1].calculate(self._tfy, self._y_pred) + tf.reduce_sum(l2_losses)
self._metric_rs = [metric(self._tfy, self._y_pred) for metric in self._metrics]
self._init_train_step(sess)
for weight in self._tf_weights:
weight *= weight_scale
if tensorboard_verbose > 0:
log_dir = os.path.join("tbLogs", str(datetime.datetime.now())[:19].replace(":", "-"))
train_dir = os.path.join(log_dir, "train")
test_dir = os.path.join(log_dir, "test")
for _dir in (log_dir, train_dir, test_dir):
if not os.path.isdir(_dir):
os.makedirs(_dir)
test_summary_ops = []
with tf.name_scope("l2_loss"):
layer_names = [
self._get_tb_name(layer) for layer in self._layers
if not isinstance(layer, SubLayer) and not isinstance(layer, ConvPoolLayer)
]
for name, l2_loss in zip(layer_names, l2_losses):
tf.summary.scalar(name, l2_loss)
with tf.name_scope("GlobalSummaries"):
test_summary_ops.append(tf.summary.scalar("loss", self._loss))
for name, metric_rs in zip(self._metric_names, self._metric_rs):
test_summary_ops.append(tf.summary.scalar(name, metric_rs))
train_merge_op = tf.summary.merge_all()
train_writer = tf.summary.FileWriter(train_dir, sess.graph)
train_writer.add_graph(sess.graph)
test_writer = tf.summary.FileWriter(test_dir)
test_merge_op = tf.summary.merge(test_summary_ops)
else:
train_writer = test_writer = train_merge_op = test_merge_op = None
args = (
x_train, y_train, x_test, y_test, show_loss,
self.verbose >= NNVerbose.METRICS_DETAIL,
tensorboard_verbose, train_repeat, sess, train_merge_op, test_merge_op,
train_writer, test_writer
)
ims = []
for counter in range(epoch):
if self.verbose >= NNVerbose.ITER and counter % record_period == 0:
sub_bar = ProgressBar(max_value=train_repeat * record_period - 1, name="Iteration")
else:
sub_bar = None
self._batch_training(
x_train, y_train, batch_size, train_repeat,
self._loss, self._train_step, sub_bar, counter, *args)
self._handle_animation(
counter, x, y, ims, animation_params, *animation_properties,
img=self._draw_2d_network(**animation_params), name="Neural Network"
)
if (counter + 1) % record_period == 0:
if do_log:
self._append_log(x_train, y_train, None, "Train", show_loss)
self._append_log(x_test, y_test, None, "Test", show_loss)
if self.verbose >= NNVerbose.METRICS:
self._print_metric_logs("Train", show_loss)
self._print_metric_logs("Test", show_loss)
if self.verbose >= NNVerbose.EPOCH:
bar.update(counter // record_period + 1)
if img is not None:
cv2.waitKey(0)
cv2.destroyAllWindows()
self._handle_mp4(ims, animation_properties, "NN")
