def __new__(mcs, *args, **kwargs):
name, bases, attr = args[:3]
timing = Timing()
for _name, _value in attr.items():
if "__" in _name or "timing" in _name or "evaluate" in _name:
continue
if Util.callable(_value):
attr[_name] = timing.timeit(level=2)(_value)
return type(name, bases, attr)
def __new__(mcs, *args, **kwargs):
name, bases, attr = args[:3]
timing = Timing()
for _name, _value in attr.items():
if "__" in _name or "timing" in _name or "estimate" in _name:
continue
if Util.callable(_value):
attr[_name] = timing.timeit(level=2)(_value)
return type(name, bases, attr)
def __new__(mcs, *args, **kwargs):
name, bases, attr = args[:3]
timing = Timing()
for _name, _value in attr.items():
if "__" in _name or "timing" in _name or "evaluate" in _name:
continue
if Util.callable(_value):
attr[_name] = timing.timeit(level=2)(_value)
def show_timing_log(self, level=2):
getattr(self, name + "Timing").show_timing_log(level)
attr["show_timing_log"] = show_timing_log
return tyep(name,bases, attr)
def __new__(mcs, *args, **kwargs):
name, bases, attr = args[:3]
timing = Timing()
for _name, _value in attr.items():
if "__" in _name or "timing" in _name or "evaluate" in _name:
continue
if Util.callable(_value):
attr[_name] = timing.timeit(level=2)(_value)
def show_timing_log(self, level=2):
getattr(self, name + "Timing").show_timing_log(level)
attr["show_timing_log"] = show_timing_log
return type(name, bases, attr)
class Layer:
LayerTiming = Timing()
def __init__(self, shape):
"""
:param shape: shape[0] = units of previous layer
shape[1] = units of current layer (self)
"""
self.shape = shape
def __str__(self):
return self.__class__.__name__
def __repr__(self):
return str(self)
@property
def name(self):
return str(self)
# Core
def _activate(self, x, predict):
pass
def derivative(self, y):
pass
@LayerTiming.timeit(level=1, prefix="[Core] ")
def activate(self, x, w, bias, predict=False):
return self._activate(x.dot(w) + bias, predict)
@LayerTiming.timeit(level=1, prefix="[Core] ")
def bp(self, y, w, prev_delta):
return prev_delta.dot(w.T) * self.derivative(y)
class Optimizers:
OptTiming = Timing()
def __init__(self, lr=1e-3):
self._lr = lr
self._opt = None
@property
def name(self):
return str(self)
def feed_timing(self, timing):
if isinstance(timing, Timing):
self.OptTiming = timing
@OptTiming.timeit(level=1, prefix="[API] ")
def minimize(self, x, *args, **kwargs):
return self._opt.minimize(x, *args, **kwargs)
def __str__(self):
return self.__class__.__name__
def __repr__(self):
return str(self)
class GDBase(ClassifierBase):
GDBaseTiming = Timing()
def __init__(self, **kwargs):
super(GDBase, self).__init__(**kwargs)
self._optimizer = self._model_parameters = self._model_grads = None
def _get_grads(self, x_batch, y_batch, y_pred, sample_weight_batch, *args):
return 0
def _update_model_params(self):
for i, (param, grad) in enumerate(
zip(self._model_parameters, self._model_grads)):
if grad is not None:
param -= self._optimizer.run(i, grad)
@GDBaseTiming.timeit(level=1, prefix="[Core] ")
def _batch_training(self, x, y, batch_size, train_repeat, *args, **kwargs):
sample_weight, *args = args
epoch_loss = 0
for i in range(train_repeat):
if train_repeat != 1:
batch = np.random.permutation(len(x))[:batch_size]
x_batch, y_batch = x[batch], y[batch]
sample_weight_batch = sample_weight[batch]
else:
x_batch, y_batch, sample_weight_batch = x, y, sample_weight
y_pred = self.predict(x_batch, get_raw_results=True, **kwargs)
epoch_loss += self._get_grads(x_batch, y_batch, y_pred,
sample_weight_batch, *args)
self._update_model_params()
self._batch_work(i, *args)
return epoch_loss / train_repeat
class LinearSVM(ClassifierBase):
LinearSVMTiming = Timing()
def __init__(self, **kwargs):
super(LinearSVM, self).__init__(**kwargs)
self._w = self._b = None
self._params["c"] = kwargs.get("c", 1)
self._params["lr"] = kwargs.get("lr", 0.001)
self._params["epoch"] = kwargs.get("epoch", 10**4)
self._params["tol"] = kwargs.get("tol", 1e-3)
@LinearSVMTiming.timeit(level=1, prefix="[API] ")
def fit(self,
x,
y,
sample_weight=None,
c=None,
lr=None,
epoch=None,
tol=None):
if sample_weight is None:
sample_weight = self._params["sw"]
if c is None:
c = self._params["c"]
if lr is None:
lr = self._params["lr"]
if epoch is None:
epoch = self._params["epoch"]
if tol is None:
tol = self._params["tol"]
x, y = np.atleast_2d(x), np.array(y)
if sample_weight is None:
sample_weight = np.ones(len(y))
else:
sample_weight = np.array(sample_weight) * len(y)
self._w = np.zeros(x.shape[1])
self._b = 0
bar = ProgressBar(max_value=epoch, name="LinearSVM")
bar.start()
for _ in range(epoch):
_err = (1 -
self.predict(x, get_raw_results=True) * y) * sample_weight
_indices = np.random.permutation(len(y))
_idx = _indices[np.argmax(_err[_indices])]
if _err[_idx] <= tol:
bar.update(epoch)
return
_delta = lr * c * y[_idx] * sample_weight[_idx]
self._w *= 1 - lr
self._w += _delta * x[_idx]
self._b += _delta
bar.update()
@LinearSVMTiming.timeit(level=1, prefix="[API] ")
def predict(self, x, get_raw_results=False):
rs = np.sum(self._w * x, axis=1) + self._b
if not get_raw_results:
return np.sign(rs)
return rs
class Optimizer:
OptTiming = Timing()
def __init__(self, lr=0.01, cache=None):
self.lr = lr
self._cache = cache
def __str__(self):
return self.__class__.__name__
def __repr__(self):
return str(self)
@property
def name(self):
return str(self)
def feed_variables(self, variables):
self._cache = [torch.zeros(var.size()) for var in variables]
@OptTiming.timeit(level=1, prefix="[API] ")
def run(self, i, dw):
return self._run(i, dw)
def _run(self, i, dw):
raise NotImplementedError(
"Please implement a 'feed' method for your optimizer")
@OptTiming.timeit(level=4, prefix="[API] ")
def update(self):
return self._update()
def _update(self):
raise NotImplementedError(
"Please implement an 'update' method for your optimizer")
class KernelPerceptron(KernelBase):
KernelPerceptronTiming = Timing()
def __init__(self, **kwargs):
super(KernelPerceptron, self).__init__(**kwargs)
self._fit_args, self._fit_args_names = [0.01], ["lr"]
@KernelPerceptronTiming.timeit(level=1, prefix="[Core] ")
def _update_dw_cache(self, idx, lr, sample_weight):
self._dw_cache = lr * self._y[idx] * sample_weight[idx]
self._w[idx] += self._dw_cache
@KernelPerceptronTiming.timeit(level=1, prefix="[Core] ")
def _update_db_cache(self, idx, lr, sample_weight):
self._db_cache = self._dw_cache
self._b += self._db_cache
@KernelPerceptronTiming.timeit(level=1, prefix="[Core] ")
def _fit(self, sample_weight, lr):
_err = (np.sign(self._prediction_cache) != self._y) * sample_weight
_indices = np.random.permutation(len(self._y))
_idx = _indices[np.argmax(_err[_indices])]
if self._prediction_cache[_idx] == self._y[_idx]:
return True
self._update_dw_cache(_idx, lr, sample_weight)
self._update_db_cache(_idx, lr, sample_weight)
self._update_pred_cache(_idx)
class TorchSVM(TorchKernelBase):
TorchSVMTiming = Timing()
def __init__(self, **kwargs):
super(TorchSVM, self).__init__(**kwargs)
self._fit_args, self._fit_args_names = [1e-3], ["tol"]
self._batch_size = kwargs.get("batch_size", 128)
self._optimizer = kwargs.get("optimizer", "Adam")
self._train_repeat = None
@TorchSVMTiming.timeit(level=1, prefix="[Core] ")
def _loss(self, y, y_pred, sample_weight):
return torch.sum(
torch.clamp(1 - y * y_pred, min=0) *
sample_weight) + 0.5 * (y_pred - self._b.expand_as(y_pred)
).unsqueeze(0).mm(self._w)
def _prepare(self, sample_weight, **kwargs):
lr = kwargs.get("lr", self._params["lr"])
self._w = Variable(torch.zeros([len(self._x), 1]),
requires_grad=True)
self._b = Variable(torch.Tensor([.0]), requires_grad=True)
self._model_parameters = [self._w, self._b]
self._optimizer = PyTorchOptFac().get_optimizer_by_name(
self._optimizer, self._model_parameters, lr,
self._params["epoch"])
sample_weight, = self._arr_to_variable(False, sample_weight)
self._loss_function = lambda y, y_pred: self._loss(
y, y_pred, sample_weight)
@TorchSVMTiming.timeit(level=1, prefix="[Core] ")
def _fit(self, sample_weight, tol):
if self._train_repeat is None:
self._train_repeat = self._get_train_repeat(
self._x, self._batch_size)
l = self.batch_training(self._gram, self._y, self._batch_size,
self._train_repeat, self._loss_function)
if l < tol:
return True
@TorchSVMTiming.timeit(level=1, prefix="[Core] ")
def _predict(self, x, get_raw_results=False, **kwargs):
if not isinstance(x, Variable):
x = Variable(torch.from_numpy(
np.asarray(x).astype(np.float32)))
rs = x.mm(self._w)
rs = rs.add_(self._b.expand_as(rs)).squeeze(1)
if get_raw_results:
return rs
return torch.sign(rs)
@TorchSVMTiming.timeit(level=1, prefix="[API] ")
def predict(self, x, get_raw_results=False, gram_provided=False):
if not gram_provided:
x = self._kernel(np.atleast_2d(x), self._x.data.numpy())
y_pred = (x.dot(self._w.data.numpy()) +
self._b.data.numpy()).ravel()
if not get_raw_results:
return np.sign(y_pred)
return y_pred
class TFClassifierBase(ClassifierBase):
"""
Tensorfow classifier framework
Implemented tensorflow ver. metrics
"""
clf_timing = Timing()
def __init__(self, **kwargs):
super(TFClassifierBase, self).__init__(**kwargs)
self._tfx = self._tfy = None
self._y_pred_raw = self._y_pred = None
self._sess = tf.Session()
@clf_timing.timeit(level=2, prefix="[Core] ")
def _batch_training(self, x, y, batch_size, train_repeat, *args):
loss, train_step, *args = args
epoch_cost = 0
for i in range(train_repeat):
if train_repeat != 1:
batch = np.random.choice(len(x), batch_size)
x_batch, y_batch = x[batch], y[batch]
else:
x_batch, y_batch = x, y
epoch_cost += self._sess.run([loss, train_step], {
self._tfx: x_batch,
self._tfy: y_batch
})[0]
self._batch_work(i, *args)
return epoch_cost / train_repeat
class NaiveBayes(ClassifierBase):
NaiveBayesTiming = Timing()
def __init__(self, **kwargs):
super(NaiveBayes, self).__init__(**kwargs)
self._x = self._y = None
self._data = self._func = None
self._n_possibilities = None
self._labelled_x = self._label_zip = None
self._cat_counter = self._con_counter = None
self.label_dic = self._feat_dics = None
self._params["lb"] = kwargs.get("lb", 1)
def feed_data(self, x, y, sample_weight=None):
pass
def feed_sample_weight(self, sample_weight=None):
pass
@NaiveBayesTiming.timeit(level=2, prefix="[API] ")
def get_prior_probability(self, lb=1):
return [(c_num + lb) / (len(self._y) + lb * len(self._cat_counter))
for c_num in self._cat_counter]
@NaiveBayesTiming.timeit(level=2, prefix="[API] ")
def fit(self, x=None, y=None, sample_weight=None, lb=None):
if sample_weight is None:
sample_weight = self._params["sw"]
if lb is None:
lb = self._params["lb"]
if x is not None and y is not None:
self.feed_data(x, y, sample_weight)
self._func = self._fit(lb)
def _fit(self, lb):
pass
@NaiveBayesTiming.timeit(level=1, prefix="[API] ")
def predict_one(self, x, get_raw_result=False):
if type(x) is np.ndarray:
x = x.tolist()
else:
x = x[:]
x = self._transfer_x(x)
m_arg, m_probability = 0, 0
for i in range(len(self._cat_counter)):
p = self._func(x, i)
if p > m_probability:
m_arg, m_probability = i, p
if not get_raw_result:
return self.label_dic[m_arg]
return m_probability
@NaiveBayesTiming.timeit(level=3, prefix="[API] ")
def predict(self, x, get_raw_result=False):
return np.array([self.predict_one(xx, get_raw_result) for xx in x])
def _transfer_x(self, x):
return x
class Perceptron(ClassifierBase, metaclass=ClassifierMeta):
PerceptronTiming = Timing()
def __init__(self):
self._w = self._b = None
@PerceptronTiming.timeit(level=1, prefix="[API] ")
def fit(self, x, y, sample_weight=None, lr=0.01, epoch=10**4):
x, y = np.atleast_2d(x), np.array(y)
if sample_weight is None:
sample_weight = np.ones(len(y))
else:
sample_weight = np.array(sample_weight) * len(y)
self._w = np.zeros(x.shape[1])
self._b = 0
for _ in range(epoch):
y_pred = self.predict(x)
_err = (y_pred != y) * sample_weight
_indices = np.random.permutation(len(y))
_idx = _indices[np.argmax(_err[_indices])]
if y_pred[_idx] == y[_idx]:
return
_delta = lr * y[_idx] * sample_weight[_idx]
self._w += _delta * x[_idx]
self._b += _delta
@PerceptronTiming.timeit(level=1, prefix="[API] ")
def predict(self, x, get_raw_results=False):
rs = np.sum(self._w * x, axis=1) + self._b
if not get_raw_results:
return np.sign(rs)
return rs
class TFLinearSVM(TFClassifierBase):
TFLinearSVMTiming = Timing()
def __init__(self, **kwargs):
super(TFLinearSVM, self).__init__(**kwargs)
self._w = self._b = None
self._params["c"] = kwargs.get("c", 1)
self._params["lr"] = kwargs.get("lr", 0.01)
self._params["batch_size"] = kwargs.get("batch_size", 128)
self._params["epoch"] = kwargs.get("epoch", 10 ** 4)
self._params["tol"] = kwargs.get("tol", 1e-3)
self._params["optimizer"] = kwargs.get("optimizer", "Adam")
@TFLinearSVMTiming.timeit(level=1, prefix="[API] ")
def fit(self, x, y, c=None, lr=None, batch_size=None, epoch=None, tol=None,
optimizer=None, animation_params=None):
if c is None:
c = self._params["c"]
if lr is None:
lr = self._params["lr"]
if batch_size is None:
batch_size = self._params["batch_size"]
if epoch is None:
epoch = self._params["epoch"]
if tol is None:
tol = self._params["tol"]
if optimizer is None:
optimizer = self._params["optimizer"]
*animation_properties, animation_params = self._get_animation_params(animation_params)
x, y = np.atleast_2d(x), np.asarray(y)
y_2d = y[..., None]
self._w = tf.Variable(np.zeros([x.shape[1], 1]), dtype=tf.float32, name="w")
self._b = tf.Variable(0., dtype=tf.float32, name="b")
self._tfx = tf.placeholder(tf.float32, [None, x.shape[1]])
self._tfy = tf.placeholder(tf.float32, [None, 1])
self._y_pred_raw = tf.matmul(self._tfx, self._w) + self._b
self._y_pred = tf.sign(self._y_pred_raw)
loss = tf.reduce_sum(
tf.nn.relu(1 - self._tfy * self._y_pred_raw)
) + c * tf.nn.l2_loss(self._w)
train_step = TFOptFac().get_optimizer_by_name(optimizer, lr).minimize(loss)
self._sess.run(tf.global_variables_initializer())
bar = ProgressBar(max_value=epoch, name="TFLinearSVM")
ims = []
train_repeat = self._get_train_repeat(x, batch_size)
for i in range(epoch):
l = self._batch_training(x, y_2d, batch_size, train_repeat, loss, train_step)
if l < tol:
bar.terminate()
break
self._handle_animation(i, x, y, ims, animation_params, *animation_properties)
bar.update()
self._handle_mp4(ims, animation_properties)
@TFLinearSVMTiming.timeit(level=1, prefix="[API] ")
def predict(self, x, get_raw_results=False, **kwargs):
rs = self._y_pred_raw if get_raw_results else self._y_pred
return self._sess.run(rs, {self._tfx: x}).ravel()
class Perceptron(ClassifierBase):
PerceptronTiming = Timing()
def __init__(self, **kwargs):
super(Perceptron, self).__init__(**kwargs)
self._w = self._b = None
self._params["lr"] = kwargs.get("lr", 0.01)
self._params["epoch"] = kwargs.get("epoch", 10**4)
@PerceptronTiming.timeit(level=1, prefix="[API] ")
def fit(self,
x,
y,
sample_weight=None,
lr=None,
epoch=None,
animation_params=None):
if sample_weight is None:
sample_weight = self._params["sample_weight"]
if lr is None:
lr = self._params["lr"]
if epoch is None:
epoch = self._params["epoch"]
*animation_properties, animation_params = self._get_animation_params(
animation_params)
x, y = np.atleast_2d(x), np.asarray(y)
if sample_weight is None:
sample_weight = np.ones(len(y))
else:
sample_weight = np.asarray(sample_weight) * len(y)
self._w = np.zeros(x.shape[1])
self._b = 0
ims = []
bar = ProgressBar(max_value=epoch, name="Perceptron")
for i in range(epoch):
y_pred = self.predict(x)
_err = (y_pred != y) * sample_weight
_indices = np.random.permutation(len(y))
_idx = _indices[np.argmax(_err[_indices])]
if y_pred[_idx] == y[_idx]:
bar.update(epoch)
break
_delta = lr * y[_idx] * sample_weight[_idx]
self._w += _delta * x[_idx]
self._b += _delta
self._handle_animation(i, x, y, ims, animation_params,
*animation_properties)
bar.update()
self._handle_mp4(ims, animation_properties)
@PerceptronTiming.timeit(level=1, prefix="[API] ")
def predict(self, x, get_raw_results=False, **kwargs):
rs = np.sum(self._w * x, axis=1) + self._b
if not get_raw_results:
return np.sign(rs)
return rs
class NaiveBayes(ClassifierBase):
NaiveBayesTiming = Timing()
def __init__(self, **kwargs):
super(NaiveBayes, self).__init__(**kwargs)
self._x = self._y = self._data = None
self._n_possibilities = self._p_category = None
self._labelled_x = self._label_zip = None
self._cat_counter = self._con_counter = None
self.label_dict = self._feat_dicts = None
self._params["lb"] = kwargs.get("lb", 1)
def feed_data(self, x, y, sample_weight=None):
pass
def feed_sample_weight(self, sample_weight=None):
pass
@NaiveBayesTiming.timeit(level=2, prefix="[API] ")
def get_prior_probability(self, lb=1):
return [(c_num + lb) / (len(self._y) + lb * len(self._cat_counter))
for c_num in self._cat_counter]
@NaiveBayesTiming.timeit(level=2, prefix="[API] ")
def fit(self, x=None, y=None, sample_weight=None, lb=None):
if sample_weight is None:
sample_weight = self._params["sample_weight"]
if lb is None:
lb = self._params["lb"]
if x is not None and y is not None:
self.feed_data(x, y, sample_weight)
self._fit(lb)
def _fit(self, lb):
pass
def _func(self, x, i):
pass
@NaiveBayesTiming.timeit(level=1, prefix="[API] ")
def predict(self, x, get_raw_result=False, **kwargs):
if isinstance(x, np.ndarray):
x = x.tolist()
else:
x = [xx[:] for xx in x]
x = self._transfer_x(x)
m_arg, m_probability = np.zeros(len(x),
dtype=np.int8), np.zeros(len(x))
for i in range(len(self._cat_counter)):
p = self._func(x, i)
mask = p > m_probability
m_arg[mask], m_probability[mask] = i, p[mask]
if not get_raw_result:
return np.array([self.label_dict[arg] for arg in m_arg])
return m_probability
def _transfer_x(self, x):
return x
class Perceptron(ClassifierBase):
PerceptronTiming = Timing()
def __init__(self, **kwargs):
super(Perceptron, self).__init__(**kwargs)
self._w = self._b = None
self._params["lr"] = kwargs.get("lr", 0.01)
self._params["epoch"] = kwargs.get("epoch", 10**4)
@PerceptronTiming.timeit(level=1, prefix="[API] ")
def fit(self,
x,
y,
sample_weight=None,
lr=None,
epoch=None,
animation_params=None):
if sample_weight is None:
sample_weight = self._params["sample_weight"]
if lr is None:
lr = self._params["lr"]
if epoch is None:
epoch = self._params["epoch"]
*animation_properties, animation_params = self._get_animation_params(
animation_params)
x, y = np.atleast_2d(x), np.asarray(y)
if sample_weight is None:
sample_weight = np.ones(len(y))
else:
sample_weight = np.asarray(sample_weight) * len(y)
self._w = np.zeros(x.shape[1])
self._b = 0.
ims = []
bar = ProgressBar(max_value=epoch, name="Perceptron")
for i in range(epoch):
err = -y * self.predict(x, True) * sample_weight
idx = np.argmax(err)
if err[idx] < 0:
bar.terminate()
break
delta = lr * y[idx] * sample_weight[idx]
self._w += delta * x[idx]
self._b += delta
self._handle_animation(i, x, y, ims, animation_params,
*animation_properties)
bar.update()
self._handle_mp4(ims, animation_properties)
@PerceptronTiming.timeit(level=1, prefix="[API] ")
def predict(self, x, get_raw_results=False, **kwargs):
rs = np.asarray(x, dtype=np.float32).dot(self._w) + self._b
if get_raw_results:
return rs
return np.sign(rs).astype(np.float32)
class CostLayer(Layer):
CostLayerTiming = Timing()
def __init__(self, parent, shape, cost_function="MSE"):
Layer.__init__(self, shape)
self._parent = parent
self._available_cost_functions = {
"MSE": CostLayer._mse,
"Cross Entropy": CostLayer._cross_entropy
}
self._cost_function_name = cost_function
self._cost_function = self._available_cost_functions[cost_function]
def __str__(self):
return self._cost_function_name
def _activate(self, x, predict):
raise ValueError("activate function should not be called in CostLayer")
def derivative(self, y):
raise ValueError(
"derivative function should not be called in CostLayer")
@CostLayerTiming.timeit(level=1, prefix="[Core] ")
def bp_first(self, y, y_pred):
if self._cost_function_name == "Cross Entropy" and (
self._parent.name == "Sigmoid" or "Softmax"):
return y - y_pred
return -self._cost_function(y,
y_pred) * self._parent.derivative(y_pred)
@property
def calculate(self):
return lambda y, y_pred: self._cost_function(y, y_pred, False)
# Cost Functions
@staticmethod
def _mse(y, y_pred, diff=True):
if diff:
return -y + y_pred
assert_string = "y or y_pred should be np.ndarray in cost function"
assert isinstance(y, np.ndarray) or isinstance(
y_pred, np.ndarray), assert_string
return 0.5 * np.average((y - y_pred)**2)
@staticmethod
def _cross_entropy(y, y_pred, diff=True, eps=1e-8):
if diff:
return -y / (y_pred + eps) + (1 - y) / (1 - y_pred + eps)
assert_string = "y or y_pred should be np.ndarray in cost function"
assert isinstance(y, np.ndarray) or isinstance(
y_pred, np.ndarray), assert_string
# noinspection PyTypeChecker
return np.average(-y * np.log(y_pred + eps) -
(1 - y) * np.log(1 - y_pred + eps))
class ConvLayer(Layer):
LayerTiming = Timing()
def __init__(self, shape, stride=1, padding="SAME", parent=None):
"""
:param shape: shape[0] = shape of previous layer c x h x w
shape[1] = shape of current layer's weight f x c x h x w
:param stride: stride
:param padding: zero-padding
:param parent: parent
"""
if parent is not None:
_parent = parent.root if parent.is_sub_layer else parent
shape, stride, padding = _parent.shape, _parent.stride, _parent.padding
Layer.__init__(self, shape)
self._stride = stride
if isinstance(padding, str):
if padding.upper() == "VALID":
self._padding = 0
self._pad_flag = "VALID"
else:
self._padding = self._pad_flag = "SAME"
else:
self._padding = int(padding)
self._pad_flag = "VALID"
self.parent = parent
if len(shape) == 1:
self.n_channels = self.n_filters = self.out_h = self.out_w = None
else:
self.feed_shape(shape)
def feed_shape(self, shape):
self.shape = shape
self.n_channels, height, width = shape[0]
self.n_filters, filter_height, filter_width = shape[1]
if self._pad_flag == "VALID":
self.out_h = ceil((height - filter_height + 1) / self._stride)
self.out_w = ceil((width - filter_width + 1) / self._stride)
else:
self.out_h = ceil(height / self._stride)
self.out_w = ceil(width / self._stride)
@property
def stride(self):
return self._stride
@property
def padding(self):
return self._padding
@property
def pad_flag(self):
return self._pad_flag
class TFSVM(KernelBase):
TFSVMTiming = Timing()
def __init__(self):
super(TFSVM, self).__init__()
self._fit_args, self._fit_args_names = [1e-3], ["tol"]
self._cost = self._train_step = None
self._sess = tf.Session()
self._do_log = False
def _prepare(self, sample_weight, **kwargs):
lr = kwargs.get("lr", self._params["lr"])
sample_weight = tf.constant(sample_weight,
dtype=tf.float32,
name="sample_weight")
x, y = tf.constant(self._x,
dtype=tf.float32), tf.constant(self._y,
dtype=tf.float32)
self._gram = tf.constant(self._kernel(self._x, self._x),
dtype=tf.float32,
name="gram")
self._w = tf.Variable(np.zeros(len(self._x))[None, ...],
dtype=tf.float32,
name="w")
self._b = tf.Variable(.0, dtype=tf.float32, name="b")
y_pred = self.predict(x, True, True)
self._cost = tf.reduce_sum(
tf.maximum(1 - y * y_pred, 0) * sample_weight) + 0.5 * tf.matmul(
self._w, tf.matmul(self._gram, self._w,
transpose_b=True))[0][0]
self._train_step = tf.train.AdamOptimizer(learning_rate=lr).minimize(
self._cost)
self._sess.run(tf.global_variables_initializer())
@TFSVMTiming.timeit(level=1, prefix="[API] ")
def _fit(self, sample_weight, tol):
_l = self._sess.run([self._cost, self._train_step])[0]
if _l < tol:
return True
@TFSVMTiming.timeit(level=1, prefix="[API] ")
def predict(self, x, get_raw_results=False, gram_provided=False):
if gram_provided:
gram = self._gram
else:
gram = tf.constant(self._kernel(np.atleast_2d(x), self._x).T,
dtype=tf.float32)
rs = tf.matmul(self._w, gram) + self._b
if not get_raw_results:
rs = tf.sign(rs)
if not gram_provided:
rs = self._sess.run(rs)
return rs
class TFSVM(TFKernelBase):
TFSVMTiming = Timing()
def __init__(self, **kwargs):
super(TFSVM, self).__init__(**kwargs)
self._fit_args, self._fit_args_names = [1e-3], ["tol"]
self._batch_size = kwargs.get("batch_size", 128)
self._optimizer = kwargs.get("optimizer", "Adam")
self._train_repeat = None
def _prepare(self, sample_weight, **kwargs):
lr = kwargs.get("lr", self._params["lr"])
self._w = tf.Variable(np.zeros([len(self._x), 1]),
dtype=tf.float32,
name="w")
self._b = tf.Variable(.0, dtype=tf.float32, name="b")
self._tfx = tf.placeholder(tf.float32, [None, None])
self._tfy = tf.placeholder(tf.float32, [None])
self._y_pred_raw = tf.transpose(
tf.matmul(self._tfx, self._w) + self._b)
self._y_pred = tf.sign(self._y_pred_raw)
self._loss = tf.reduce_sum(
tf.maximum(1 - self._tfy * self._y_pred_raw, 0) *
sample_weight) + 0.5 * tf.matmul(
# self._w, tf.matmul(self._tfx, self._w, transpose_b=True)
(self._y_pred_raw - self._b),
self._w)[0][0]
self._train_step = TFOptFac().get_optimizer_by_name(
self._optimizer, lr).minimize(self._loss)
self._sess.run(tf.global_variables_initializer())
@TFSVMTiming.timeit(level=1, prefix="[API] ")
def _fit(self, sample_weight, tol):
if self._train_repeat is None:
self._train_repeat = self._get_train_repeat(
self._x, self._batch_size)
l = self._batch_training(self._gram, self._y, self._batch_size,
self._train_repeat, self._loss,
self._train_step)
if l < tol:
return True
@TFSVMTiming.timeit(level=1, prefix="[API] ")
def predict(self, x, get_raw_results=False, gram_provided=False):
rs = self._y_pred_raw if get_raw_results else self._y_pred
if gram_provided:
return self._sess.run(rs, {self._tfx: x}).ravel()
return self._sess.run(
rs, {
self._tfx: self._kernel(np.atleast_2d(x), self._x)
}).ravel()
class TFLinearSVM(ClassifierBase):
TFLinearSVMTiming = Timing()
def __init__(self, **kwargs):
super(TFLinearSVM, self).__init__(**kwargs)
self._w = self._b = None
self._sess = tf.Session()
self._params["c"] = kwargs.get("c", 1)
self._params["lr"] = kwargs.get("lr", 0.001)
self._params["epoch"] = kwargs.get("epoch", 10 ** 4)
self._params["tol"] = kwargs.get("tol", 1e-3)
@TFLinearSVMTiming.timeit(level=1, prefix="[API] ")
def fit(self, x, y, sample_weight=None, c=None, lr=None, epoch=None, tol=None):
if sample_weight is None:
sample_weight = self._params["sw"]
if c is None:
c = self._params["c"]
if lr is None:
lr = self._params["lr"]
if epoch is None:
epoch = self._params["epoch"]
if tol is None:
tol = self._params["tol"]
if sample_weight is None:
sample_weight = tf.constant(np.ones(len(y)), dtype=tf.float32, name="sample_weight")
else:
sample_weight = tf.constant(np.asarray(sample_weight) * len(y), dtype=tf.float32, name="sample_weight")
x, y = tf.constant(x, dtype=tf.float32), tf.constant(y, dtype=tf.float32)
self._w = tf.Variable(np.zeros(x.shape[1]), dtype=tf.float32, name="w")
self._b = tf.Variable(0., dtype=tf.float32, name="b")
y_pred = self.predict(x, True, False)
cost = tf.reduce_sum(tf.maximum(1 - y * y_pred, 0) * sample_weight) + c * tf.nn.l2_loss(self._w)
train_step = tf.train.AdamOptimizer(learning_rate=lr).minimize(cost)
self._sess.run(tf.global_variables_initializer())
bar = ProgressBar(max_value=epoch, name="TFLinearSVM")
for _ in range(epoch):
_l = self._sess.run([cost, train_step])[0]
if _l < tol:
bar.update(epoch)
break
bar.update()
@TFLinearSVMTiming.timeit(level=1, prefix="[API] ")
def predict(self, x, get_raw_results=False, out_of_sess=True):
rs = tf.reduce_sum(self._w * x, axis=1) + self._b
if not get_raw_results:
rs = tf.sign(rs)
if out_of_sess:
rs = self._sess.run(rs)
return rs
class T1234(ClassifierBase, metaclass=ClassifierMeta):
T1234Timing = Timing()
@staticmethod
@T1234Timing.timeit(level=1, prefix="[API] ")
def predict(x):
y_pred = np.zeros(len(x))
x_axis, y_axis = x.T
x_lt_0, y_lt_0 = x_axis < 0, y_axis < 0
x_gt_0, y_gt_0 = ~x_lt_0, ~y_lt_0
y_pred[x_lt_0 & y_gt_0] = 1
y_pred[x_lt_0 & y_lt_0] = 2
y_pred[x_gt_0 & y_gt_0] = 3
return y_pred
class T1234(ClassifierBase):
T1234Timing = Timing()
@staticmethod
@T1234Timing.timeit(level=1, prefix="[API] ")
def predict(self, x, get_raw_results=False):
y_pred = np.zeros(len(x))
x_axis, y_axis = x.T
x_lt_0, y_lt_0 = x_axis < 0, y_axis < 0
x_gt_0, y_gt_0 = ~x_lt_0, ~y_lt_0
y_pred[x_lt_0 & y_gt_0] = 1
y_pred[x_lt_0 & y_lt_0] = 2
y_pred[x_gt_0 & y_gt_0] = 3
return y_pred
class NaiveBayes(ClassifierBase, metaclass=ClassifierMeta):
NaiveBayesTiming = Timing()
def __init__(self):
self._x = self._y = None
self._data = self._func = None
self._n_possibilities = None
self._labelled_x = self._label_zip = None
self._cat_counter = self._con_counter = None
self.label_dic = self._feat_dics = None
def feed_data(self, x, y, sample_weight=None):
pass
def _feed_sample_weight(self, sample_weight=None):
pass
@NaiveBayesTiming.timeit(level=2, prefix="[API] ")
def get_prior_probability(self, lb=1):
return [(_c_num + lb) / (len(self._y) + lb * len(self._cat_counter))
for _c_num in self._cat_counter]
@NaiveBayesTiming.timeit(level=2, prefix="[API] ")
def fit(self, x=None, y=None, sample_weight=None, lb=1):
if x is not None and y is not None:
self.feed_data(x, y, sample_weight)
self._func = self._fit(lb)
def _fit(self, lb):
pass
@NaiveBayesTiming.timeit(level=1, prefix="[API] ")
def predict(self, x, get_raw_result=False):
if isinstance(x, np.ndarray):
x = x.tolist()
else:
x = [xx[:] for xx in x]
x = self._transfer_x(x)
m_arg, m_probability = np.zeros(len(x),
dtype=np.int8), np.zeros(len(x))
for i in range(len(self._cat_counter)):
p = self._func(x, i)
_mask = p > m_probability
m_arg[_mask], m_probability[_mask] = i, p[_mask]
if not get_raw_result:
return np.array([self.label_dic[arg] for arg in m_arg])
return m_probability
def _transfer_x(self, x):
return x
class TFClassifierBase(ClassifierBase):
"""
Tensorfow classifier framework
Implemented tensorflow ver. metrics
"""
clf_timing = Timing()
def __init__(self, **kwargs):
super(TFClassifierBase, self).__init__(**kwargs)
self._tfx = self._tfy = None
self._y_pred_raw = self._y_pred = None
self._sess = tf.Session()
@staticmethod
@clf_timing.timeit(level=2, prefix="[Metric] ")
def acc(y, y_pred, weights=None):
y_arg, y_pred_arg = tf.argmax(y, axis=1), tf.argmax(y_pred, axis=1)
same = tf.cast(tf.equal(y_arg, y_pred_arg), tf.float32)
if weights is not None:
same *= weights
return tf.reduce_mean(same)
@staticmethod
@clf_timing.timeit(level=2, prefix="[Metric] ")
def f1_score(y, y_pred):
y_arg, y_pred_arg = tf.argmax(y, axis=1), tf.argmax(y_pred, axis=1)
tp = tf.reduce_sum(y_arg * y_pred_arg)
if tp == 0:
return .0
fp = tf.reduce_sum((1 - y_arg) * y_pred_arg)
fn = tf.reduce_sum(y_arg * (1 - y_pred_arg))
return 2 * tp / (2 * tp + fn + fp)
@clf_timing.timeit(level=2, prefix="[Core] ")
def _batch_training(self, x, y, batch_size, train_repeat, *args):
loss, train_step, *args = args
epoch_cost = 0
for i in range(train_repeat):
if train_repeat != 1:
batch = np.random.choice(len(x), batch_size)
x_batch, y_batch = x[batch], y[batch]
else:
x_batch, y_batch = x, y
epoch_cost += self._sess.run([loss, train_step], {
self._tfx: x_batch,
self._tfy: y_batch
})[0]
self._batch_work(i, *args)
return epoch_cost / train_repeat
def vis_test():
nn = NNDist()
epoch = 1000
record_period = 4
make_mp4 = True
timing = Timing(enabled=True)
timing_level = 1
x, y = DataUtil.gen_spiral(50, 3, 3, 2.5)
nn.build([x.shape[1], 6, 6, 6, y.shape[1]])
nn.optimizer = "Adam"
nn.preview()
nn.feed_timing(timing)
nn.fit(x,
y,
verbose=1,
record_period=record_period,
epoch=epoch,
train_only=True,
draw_detailed_network=True,
make_mp4=make_mp4,
show_animation=True)
nn.draw_results()
timing.show_timing_log(timing_level)
class GDKP(GDKernelBase):
GDKPTiming = Timing()
@GDKPTiming.timeit(level=1, prefix="[Core] ")
def _get_grads(self, x_batch, y_batch, y_pred, sample_weight_batch, *args):
err = -y_batch * (x_batch.dot(self._alpha) + self._b) * sample_weight_batch
mask = err >= 0 # type: np.ndarray
if not np.any(mask):
self._model_grads = [None, None]
else:
delta = -y_batch[mask] * sample_weight_batch[mask]
self._model_grads = [
np.sum(delta[..., None] * x_batch[mask], axis=0),
np.sum(delta)
]
return np.sum(err[mask])
class GDSVM(GDKernelBase):
GDSVMTiming = Timing()
@GDSVMTiming.timeit(level=1, prefix="[Core] ")
def _get_grads(self, x_batch, y_batch, y_pred, sample_weight_batch, *args):
err = -y_batch * (x_batch.dot(self._alpha) + self._b)
mask = err >= 0
if np.max(err) < 0:
self._model_grads = [None, None]
else:
delta = -y_batch[mask] * sample_weight_batch[mask]
self._model_grads = [
np.sum(delta[..., None] * x_batch[mask], axis=0),
np.sum(delta)
]
return np.sum(err[mask]) + 0.5 * (y_pred - self._b).dot(self._alpha)
class ConvLayer(Layer):
LayerTiming = Timing()
def __init__(self, shape, stride=1, padding=None, parent=None):
"""
:param shape: shape[0] = shape of previous layer c x h x w
shape[1] = shape of current layer's weight f x h x w
:param stride: stride
:param padding: zero-padding
:param parent: parent
"""
if parent is not None:
_parent = parent.root if parent.is_sub_layer else parent
shape = _parent.shape
Layer.__init__(self, shape)
self.stride = stride
if padding is None:
padding = "SAME"
if isinstance(padding, str):
if padding.upper() == "VALID":
self.padding = 0
self.pad_flag = "VALID"
else:
self.padding = self.pad_flag = "SAME"
elif isinstance(padding, int):
self.padding = padding
self.pad_flag = "VALID"
else:
raise ValueError("Padding should be 'SAME' or 'VALID' or integer")
self.parent = parent
if len(shape) == 1:
self.n_channels = self.n_filters = self.out_h = self.out_w = None
else:
self.feed_shape(shape)
def feed_shape(self, shape):
self.shape = shape
self.n_channels, height, width = shape[0]
self.n_filters, filter_height, filter_width = shape[1]
if self.pad_flag == "VALID":
self.out_h = ceil((height - filter_height + 1) / self.stride)
self.out_w = ceil((width - filter_width + 1) / self.stride)
else:
self.out_h = ceil(height / self.stride)
self.out_w = ceil(width / self.stride)
class GDKernelBase(KernelBase, GDBase):
""" Kernel classifier with gradient descent algorithm """
GDKernelBaseTiming = Timing()
def __init__(self, **kwargs):
super(GDKernelBase, self).__init__(**kwargs)
self._fit_args, self._fit_args_names = [1e-3], ["tol"]
self._batch_size = kwargs.get("batch_size", 128)
self._optimizer = kwargs.get("optimizer", "Adam")
self._train_repeat = 0
def _prepare(self, sample_weight, **kwargs):
lr = kwargs.get("lr", self._params["lr"])
self._alpha = np.random.random(len(self._x)).astype(np.float32)
self._b = np.random.random(1).astype(np.float32)
self._model_parameters = [self._alpha, self._b]
self._optimizer = OptFactory().get_optimizer_by_name(
self._optimizer, self._model_parameters, lr, self._params["epoch"])
@GDKernelBaseTiming.timeit(level=1, prefix="[Core] ")
def _fit(self, sample_weight, tol):
if self._train_repeat == 0:
self._train_repeat = self._get_train_repeat(
self._x, self._batch_size)
l = self._batch_training(self._gram,
self._y,
self._batch_size,
self._train_repeat,
sample_weight,
gram_provided=True)
if l < tol:
return True
@GDKernelBaseTiming.timeit(level=1, prefix="[API] ")
def predict(self, x, get_raw_results=False, gram_provided=False):
if not gram_provided:
x = self._kernel(self._x, np.atleast_2d(x))
elif self._alpha.shape[0] != x.shape[0]:
x = x.T
y_pred = (self._alpha.dot(x) + self._b).ravel()
if not get_raw_results:
return np.sign(y_pred)
return y_pred
