def __new__(mcs, *args, **kwargs): name, bases, attr = args[:3] timing = Timing() for _name, _value in attr.items(): if "__" in _name or "timing" in _name or "evaluate" in _name: continue if Util.callable(_value): attr[_name] = timing.timeit(level=2)(_value) return type(name, bases, attr)
def __new__(mcs, *args, **kwargs): name, bases, attr = args[:3] timing = Timing() for _name, _value in attr.items(): if "__" in _name or "timing" in _name or "estimate" in _name: continue if Util.callable(_value): attr[_name] = timing.timeit(level=2)(_value) return type(name, bases, attr)
def __new__(mcs, *args, **kwargs): name, bases, attr = args[:3] timing = Timing() for _name, _value in attr.items(): if "__" in _name or "timing" in _name or "evaluate" in _name: continue if Util.callable(_value): attr[_name] = timing.timeit(level=2)(_value) def show_timing_log(self, level=2): getattr(self, name + "Timing").show_timing_log(level) attr["show_timing_log"] = show_timing_log return tyep(name,bases, attr)
def __new__(mcs, *args, **kwargs): name, bases, attr = args[:3] timing = Timing() for _name, _value in attr.items(): if "__" in _name or "timing" in _name or "evaluate" in _name: continue if Util.callable(_value): attr[_name] = timing.timeit(level=2)(_value) def show_timing_log(self, level=2): getattr(self, name + "Timing").show_timing_log(level) attr["show_timing_log"] = show_timing_log return type(name, bases, attr)
class Layer: LayerTiming = Timing() def __init__(self, shape): """ :param shape: shape[0] = units of previous layer shape[1] = units of current layer (self) """ self.shape = shape def __str__(self): return self.__class__.__name__ def __repr__(self): return str(self) @property def name(self): return str(self) # Core def _activate(self, x, predict): pass def derivative(self, y): pass @LayerTiming.timeit(level=1, prefix="[Core] ") def activate(self, x, w, bias, predict=False): return self._activate(x.dot(w) + bias, predict) @LayerTiming.timeit(level=1, prefix="[Core] ") def bp(self, y, w, prev_delta): return prev_delta.dot(w.T) * self.derivative(y)
class Optimizers: OptTiming = Timing() def __init__(self, lr=1e-3): self._lr = lr self._opt = None @property def name(self): return str(self) def feed_timing(self, timing): if isinstance(timing, Timing): self.OptTiming = timing @OptTiming.timeit(level=1, prefix="[API] ") def minimize(self, x, *args, **kwargs): return self._opt.minimize(x, *args, **kwargs) def __str__(self): return self.__class__.__name__ def __repr__(self): return str(self)
class GDBase(ClassifierBase): GDBaseTiming = Timing() def __init__(self, **kwargs): super(GDBase, self).__init__(**kwargs) self._optimizer = self._model_parameters = self._model_grads = None def _get_grads(self, x_batch, y_batch, y_pred, sample_weight_batch, *args): return 0 def _update_model_params(self): for i, (param, grad) in enumerate( zip(self._model_parameters, self._model_grads)): if grad is not None: param -= self._optimizer.run(i, grad) @GDBaseTiming.timeit(level=1, prefix="[Core] ") def _batch_training(self, x, y, batch_size, train_repeat, *args, **kwargs): sample_weight, *args = args epoch_loss = 0 for i in range(train_repeat): if train_repeat != 1: batch = np.random.permutation(len(x))[:batch_size] x_batch, y_batch = x[batch], y[batch] sample_weight_batch = sample_weight[batch] else: x_batch, y_batch, sample_weight_batch = x, y, sample_weight y_pred = self.predict(x_batch, get_raw_results=True, **kwargs) epoch_loss += self._get_grads(x_batch, y_batch, y_pred, sample_weight_batch, *args) self._update_model_params() self._batch_work(i, *args) return epoch_loss / train_repeat
class LinearSVM(ClassifierBase): LinearSVMTiming = Timing() def __init__(self, **kwargs): super(LinearSVM, self).__init__(**kwargs) self._w = self._b = None self._params["c"] = kwargs.get("c", 1) self._params["lr"] = kwargs.get("lr", 0.001) self._params["epoch"] = kwargs.get("epoch", 10**4) self._params["tol"] = kwargs.get("tol", 1e-3) @LinearSVMTiming.timeit(level=1, prefix="[API] ") def fit(self, x, y, sample_weight=None, c=None, lr=None, epoch=None, tol=None): if sample_weight is None: sample_weight = self._params["sw"] if c is None: c = self._params["c"] if lr is None: lr = self._params["lr"] if epoch is None: epoch = self._params["epoch"] if tol is None: tol = self._params["tol"] x, y = np.atleast_2d(x), np.array(y) if sample_weight is None: sample_weight = np.ones(len(y)) else: sample_weight = np.array(sample_weight) * len(y) self._w = np.zeros(x.shape[1]) self._b = 0 bar = ProgressBar(max_value=epoch, name="LinearSVM") bar.start() for _ in range(epoch): _err = (1 - self.predict(x, get_raw_results=True) * y) * sample_weight _indices = np.random.permutation(len(y)) _idx = _indices[np.argmax(_err[_indices])] if _err[_idx] <= tol: bar.update(epoch) return _delta = lr * c * y[_idx] * sample_weight[_idx] self._w *= 1 - lr self._w += _delta * x[_idx] self._b += _delta bar.update() @LinearSVMTiming.timeit(level=1, prefix="[API] ") def predict(self, x, get_raw_results=False): rs = np.sum(self._w * x, axis=1) + self._b if not get_raw_results: return np.sign(rs) return rs
class Optimizer: OptTiming = Timing() def __init__(self, lr=0.01, cache=None): self.lr = lr self._cache = cache def __str__(self): return self.__class__.__name__ def __repr__(self): return str(self) @property def name(self): return str(self) def feed_variables(self, variables): self._cache = [torch.zeros(var.size()) for var in variables] @OptTiming.timeit(level=1, prefix="[API] ") def run(self, i, dw): return self._run(i, dw) def _run(self, i, dw): raise NotImplementedError( "Please implement a 'feed' method for your optimizer") @OptTiming.timeit(level=4, prefix="[API] ") def update(self): return self._update() def _update(self): raise NotImplementedError( "Please implement an 'update' method for your optimizer")
class KernelPerceptron(KernelBase): KernelPerceptronTiming = Timing() def __init__(self, **kwargs): super(KernelPerceptron, self).__init__(**kwargs) self._fit_args, self._fit_args_names = [0.01], ["lr"] @KernelPerceptronTiming.timeit(level=1, prefix="[Core] ") def _update_dw_cache(self, idx, lr, sample_weight): self._dw_cache = lr * self._y[idx] * sample_weight[idx] self._w[idx] += self._dw_cache @KernelPerceptronTiming.timeit(level=1, prefix="[Core] ") def _update_db_cache(self, idx, lr, sample_weight): self._db_cache = self._dw_cache self._b += self._db_cache @KernelPerceptronTiming.timeit(level=1, prefix="[Core] ") def _fit(self, sample_weight, lr): _err = (np.sign(self._prediction_cache) != self._y) * sample_weight _indices = np.random.permutation(len(self._y)) _idx = _indices[np.argmax(_err[_indices])] if self._prediction_cache[_idx] == self._y[_idx]: return True self._update_dw_cache(_idx, lr, sample_weight) self._update_db_cache(_idx, lr, sample_weight) self._update_pred_cache(_idx)
class TorchSVM(TorchKernelBase): TorchSVMTiming = Timing() def __init__(self, **kwargs): super(TorchSVM, self).__init__(**kwargs) self._fit_args, self._fit_args_names = [1e-3], ["tol"] self._batch_size = kwargs.get("batch_size", 128) self._optimizer = kwargs.get("optimizer", "Adam") self._train_repeat = None @TorchSVMTiming.timeit(level=1, prefix="[Core] ") def _loss(self, y, y_pred, sample_weight): return torch.sum( torch.clamp(1 - y * y_pred, min=0) * sample_weight) + 0.5 * (y_pred - self._b.expand_as(y_pred) ).unsqueeze(0).mm(self._w) def _prepare(self, sample_weight, **kwargs): lr = kwargs.get("lr", self._params["lr"]) self._w = Variable(torch.zeros([len(self._x), 1]), requires_grad=True) self._b = Variable(torch.Tensor([.0]), requires_grad=True) self._model_parameters = [self._w, self._b] self._optimizer = PyTorchOptFac().get_optimizer_by_name( self._optimizer, self._model_parameters, lr, self._params["epoch"]) sample_weight, = self._arr_to_variable(False, sample_weight) self._loss_function = lambda y, y_pred: self._loss( y, y_pred, sample_weight) @TorchSVMTiming.timeit(level=1, prefix="[Core] ") def _fit(self, sample_weight, tol): if self._train_repeat is None: self._train_repeat = self._get_train_repeat( self._x, self._batch_size) l = self.batch_training(self._gram, self._y, self._batch_size, self._train_repeat, self._loss_function) if l < tol: return True @TorchSVMTiming.timeit(level=1, prefix="[Core] ") def _predict(self, x, get_raw_results=False, **kwargs): if not isinstance(x, Variable): x = Variable(torch.from_numpy( np.asarray(x).astype(np.float32))) rs = x.mm(self._w) rs = rs.add_(self._b.expand_as(rs)).squeeze(1) if get_raw_results: return rs return torch.sign(rs) @TorchSVMTiming.timeit(level=1, prefix="[API] ") def predict(self, x, get_raw_results=False, gram_provided=False): if not gram_provided: x = self._kernel(np.atleast_2d(x), self._x.data.numpy()) y_pred = (x.dot(self._w.data.numpy()) + self._b.data.numpy()).ravel() if not get_raw_results: return np.sign(y_pred) return y_pred
class TFClassifierBase(ClassifierBase): """ Tensorfow classifier framework Implemented tensorflow ver. metrics """ clf_timing = Timing() def __init__(self, **kwargs): super(TFClassifierBase, self).__init__(**kwargs) self._tfx = self._tfy = None self._y_pred_raw = self._y_pred = None self._sess = tf.Session() @clf_timing.timeit(level=2, prefix="[Core] ") def _batch_training(self, x, y, batch_size, train_repeat, *args): loss, train_step, *args = args epoch_cost = 0 for i in range(train_repeat): if train_repeat != 1: batch = np.random.choice(len(x), batch_size) x_batch, y_batch = x[batch], y[batch] else: x_batch, y_batch = x, y epoch_cost += self._sess.run([loss, train_step], { self._tfx: x_batch, self._tfy: y_batch })[0] self._batch_work(i, *args) return epoch_cost / train_repeat
class NaiveBayes(ClassifierBase): NaiveBayesTiming = Timing() def __init__(self, **kwargs): super(NaiveBayes, self).__init__(**kwargs) self._x = self._y = None self._data = self._func = None self._n_possibilities = None self._labelled_x = self._label_zip = None self._cat_counter = self._con_counter = None self.label_dic = self._feat_dics = None self._params["lb"] = kwargs.get("lb", 1) def feed_data(self, x, y, sample_weight=None): pass def feed_sample_weight(self, sample_weight=None): pass @NaiveBayesTiming.timeit(level=2, prefix="[API] ") def get_prior_probability(self, lb=1): return [(c_num + lb) / (len(self._y) + lb * len(self._cat_counter)) for c_num in self._cat_counter] @NaiveBayesTiming.timeit(level=2, prefix="[API] ") def fit(self, x=None, y=None, sample_weight=None, lb=None): if sample_weight is None: sample_weight = self._params["sw"] if lb is None: lb = self._params["lb"] if x is not None and y is not None: self.feed_data(x, y, sample_weight) self._func = self._fit(lb) def _fit(self, lb): pass @NaiveBayesTiming.timeit(level=1, prefix="[API] ") def predict_one(self, x, get_raw_result=False): if type(x) is np.ndarray: x = x.tolist() else: x = x[:] x = self._transfer_x(x) m_arg, m_probability = 0, 0 for i in range(len(self._cat_counter)): p = self._func(x, i) if p > m_probability: m_arg, m_probability = i, p if not get_raw_result: return self.label_dic[m_arg] return m_probability @NaiveBayesTiming.timeit(level=3, prefix="[API] ") def predict(self, x, get_raw_result=False): return np.array([self.predict_one(xx, get_raw_result) for xx in x]) def _transfer_x(self, x): return x
class Perceptron(ClassifierBase, metaclass=ClassifierMeta): PerceptronTiming = Timing() def __init__(self): self._w = self._b = None @PerceptronTiming.timeit(level=1, prefix="[API] ") def fit(self, x, y, sample_weight=None, lr=0.01, epoch=10**4): x, y = np.atleast_2d(x), np.array(y) if sample_weight is None: sample_weight = np.ones(len(y)) else: sample_weight = np.array(sample_weight) * len(y) self._w = np.zeros(x.shape[1]) self._b = 0 for _ in range(epoch): y_pred = self.predict(x) _err = (y_pred != y) * sample_weight _indices = np.random.permutation(len(y)) _idx = _indices[np.argmax(_err[_indices])] if y_pred[_idx] == y[_idx]: return _delta = lr * y[_idx] * sample_weight[_idx] self._w += _delta * x[_idx] self._b += _delta @PerceptronTiming.timeit(level=1, prefix="[API] ") def predict(self, x, get_raw_results=False): rs = np.sum(self._w * x, axis=1) + self._b if not get_raw_results: return np.sign(rs) return rs
class TFLinearSVM(TFClassifierBase): TFLinearSVMTiming = Timing() def __init__(self, **kwargs): super(TFLinearSVM, self).__init__(**kwargs) self._w = self._b = None self._params["c"] = kwargs.get("c", 1) self._params["lr"] = kwargs.get("lr", 0.01) self._params["batch_size"] = kwargs.get("batch_size", 128) self._params["epoch"] = kwargs.get("epoch", 10 ** 4) self._params["tol"] = kwargs.get("tol", 1e-3) self._params["optimizer"] = kwargs.get("optimizer", "Adam") @TFLinearSVMTiming.timeit(level=1, prefix="[API] ") def fit(self, x, y, c=None, lr=None, batch_size=None, epoch=None, tol=None, optimizer=None, animation_params=None): if c is None: c = self._params["c"] if lr is None: lr = self._params["lr"] if batch_size is None: batch_size = self._params["batch_size"] if epoch is None: epoch = self._params["epoch"] if tol is None: tol = self._params["tol"] if optimizer is None: optimizer = self._params["optimizer"] *animation_properties, animation_params = self._get_animation_params(animation_params) x, y = np.atleast_2d(x), np.asarray(y) y_2d = y[..., None] self._w = tf.Variable(np.zeros([x.shape[1], 1]), dtype=tf.float32, name="w") self._b = tf.Variable(0., dtype=tf.float32, name="b") self._tfx = tf.placeholder(tf.float32, [None, x.shape[1]]) self._tfy = tf.placeholder(tf.float32, [None, 1]) self._y_pred_raw = tf.matmul(self._tfx, self._w) + self._b self._y_pred = tf.sign(self._y_pred_raw) loss = tf.reduce_sum( tf.nn.relu(1 - self._tfy * self._y_pred_raw) ) + c * tf.nn.l2_loss(self._w) train_step = TFOptFac().get_optimizer_by_name(optimizer, lr).minimize(loss) self._sess.run(tf.global_variables_initializer()) bar = ProgressBar(max_value=epoch, name="TFLinearSVM") ims = [] train_repeat = self._get_train_repeat(x, batch_size) for i in range(epoch): l = self._batch_training(x, y_2d, batch_size, train_repeat, loss, train_step) if l < tol: bar.terminate() break self._handle_animation(i, x, y, ims, animation_params, *animation_properties) bar.update() self._handle_mp4(ims, animation_properties) @TFLinearSVMTiming.timeit(level=1, prefix="[API] ") def predict(self, x, get_raw_results=False, **kwargs): rs = self._y_pred_raw if get_raw_results else self._y_pred return self._sess.run(rs, {self._tfx: x}).ravel()
class Perceptron(ClassifierBase): PerceptronTiming = Timing() def __init__(self, **kwargs): super(Perceptron, self).__init__(**kwargs) self._w = self._b = None self._params["lr"] = kwargs.get("lr", 0.01) self._params["epoch"] = kwargs.get("epoch", 10**4) @PerceptronTiming.timeit(level=1, prefix="[API] ") def fit(self, x, y, sample_weight=None, lr=None, epoch=None, animation_params=None): if sample_weight is None: sample_weight = self._params["sample_weight"] if lr is None: lr = self._params["lr"] if epoch is None: epoch = self._params["epoch"] *animation_properties, animation_params = self._get_animation_params( animation_params) x, y = np.atleast_2d(x), np.asarray(y) if sample_weight is None: sample_weight = np.ones(len(y)) else: sample_weight = np.asarray(sample_weight) * len(y) self._w = np.zeros(x.shape[1]) self._b = 0 ims = [] bar = ProgressBar(max_value=epoch, name="Perceptron") for i in range(epoch): y_pred = self.predict(x) _err = (y_pred != y) * sample_weight _indices = np.random.permutation(len(y)) _idx = _indices[np.argmax(_err[_indices])] if y_pred[_idx] == y[_idx]: bar.update(epoch) break _delta = lr * y[_idx] * sample_weight[_idx] self._w += _delta * x[_idx] self._b += _delta self._handle_animation(i, x, y, ims, animation_params, *animation_properties) bar.update() self._handle_mp4(ims, animation_properties) @PerceptronTiming.timeit(level=1, prefix="[API] ") def predict(self, x, get_raw_results=False, **kwargs): rs = np.sum(self._w * x, axis=1) + self._b if not get_raw_results: return np.sign(rs) return rs
class NaiveBayes(ClassifierBase): NaiveBayesTiming = Timing() def __init__(self, **kwargs): super(NaiveBayes, self).__init__(**kwargs) self._x = self._y = self._data = None self._n_possibilities = self._p_category = None self._labelled_x = self._label_zip = None self._cat_counter = self._con_counter = None self.label_dict = self._feat_dicts = None self._params["lb"] = kwargs.get("lb", 1) def feed_data(self, x, y, sample_weight=None): pass def feed_sample_weight(self, sample_weight=None): pass @NaiveBayesTiming.timeit(level=2, prefix="[API] ") def get_prior_probability(self, lb=1): return [(c_num + lb) / (len(self._y) + lb * len(self._cat_counter)) for c_num in self._cat_counter] @NaiveBayesTiming.timeit(level=2, prefix="[API] ") def fit(self, x=None, y=None, sample_weight=None, lb=None): if sample_weight is None: sample_weight = self._params["sample_weight"] if lb is None: lb = self._params["lb"] if x is not None and y is not None: self.feed_data(x, y, sample_weight) self._fit(lb) def _fit(self, lb): pass def _func(self, x, i): pass @NaiveBayesTiming.timeit(level=1, prefix="[API] ") def predict(self, x, get_raw_result=False, **kwargs): if isinstance(x, np.ndarray): x = x.tolist() else: x = [xx[:] for xx in x] x = self._transfer_x(x) m_arg, m_probability = np.zeros(len(x), dtype=np.int8), np.zeros(len(x)) for i in range(len(self._cat_counter)): p = self._func(x, i) mask = p > m_probability m_arg[mask], m_probability[mask] = i, p[mask] if not get_raw_result: return np.array([self.label_dict[arg] for arg in m_arg]) return m_probability def _transfer_x(self, x): return x
class Perceptron(ClassifierBase): PerceptronTiming = Timing() def __init__(self, **kwargs): super(Perceptron, self).__init__(**kwargs) self._w = self._b = None self._params["lr"] = kwargs.get("lr", 0.01) self._params["epoch"] = kwargs.get("epoch", 10**4) @PerceptronTiming.timeit(level=1, prefix="[API] ") def fit(self, x, y, sample_weight=None, lr=None, epoch=None, animation_params=None): if sample_weight is None: sample_weight = self._params["sample_weight"] if lr is None: lr = self._params["lr"] if epoch is None: epoch = self._params["epoch"] *animation_properties, animation_params = self._get_animation_params( animation_params) x, y = np.atleast_2d(x), np.asarray(y) if sample_weight is None: sample_weight = np.ones(len(y)) else: sample_weight = np.asarray(sample_weight) * len(y) self._w = np.zeros(x.shape[1]) self._b = 0. ims = [] bar = ProgressBar(max_value=epoch, name="Perceptron") for i in range(epoch): err = -y * self.predict(x, True) * sample_weight idx = np.argmax(err) if err[idx] < 0: bar.terminate() break delta = lr * y[idx] * sample_weight[idx] self._w += delta * x[idx] self._b += delta self._handle_animation(i, x, y, ims, animation_params, *animation_properties) bar.update() self._handle_mp4(ims, animation_properties) @PerceptronTiming.timeit(level=1, prefix="[API] ") def predict(self, x, get_raw_results=False, **kwargs): rs = np.asarray(x, dtype=np.float32).dot(self._w) + self._b if get_raw_results: return rs return np.sign(rs).astype(np.float32)
class CostLayer(Layer): CostLayerTiming = Timing() def __init__(self, parent, shape, cost_function="MSE"): Layer.__init__(self, shape) self._parent = parent self._available_cost_functions = { "MSE": CostLayer._mse, "Cross Entropy": CostLayer._cross_entropy } self._cost_function_name = cost_function self._cost_function = self._available_cost_functions[cost_function] def __str__(self): return self._cost_function_name def _activate(self, x, predict): raise ValueError("activate function should not be called in CostLayer") def derivative(self, y): raise ValueError( "derivative function should not be called in CostLayer") @CostLayerTiming.timeit(level=1, prefix="[Core] ") def bp_first(self, y, y_pred): if self._cost_function_name == "Cross Entropy" and ( self._parent.name == "Sigmoid" or "Softmax"): return y - y_pred return -self._cost_function(y, y_pred) * self._parent.derivative(y_pred) @property def calculate(self): return lambda y, y_pred: self._cost_function(y, y_pred, False) # Cost Functions @staticmethod def _mse(y, y_pred, diff=True): if diff: return -y + y_pred assert_string = "y or y_pred should be np.ndarray in cost function" assert isinstance(y, np.ndarray) or isinstance( y_pred, np.ndarray), assert_string return 0.5 * np.average((y - y_pred)**2) @staticmethod def _cross_entropy(y, y_pred, diff=True, eps=1e-8): if diff: return -y / (y_pred + eps) + (1 - y) / (1 - y_pred + eps) assert_string = "y or y_pred should be np.ndarray in cost function" assert isinstance(y, np.ndarray) or isinstance( y_pred, np.ndarray), assert_string # noinspection PyTypeChecker return np.average(-y * np.log(y_pred + eps) - (1 - y) * np.log(1 - y_pred + eps))
class ConvLayer(Layer): LayerTiming = Timing() def __init__(self, shape, stride=1, padding="SAME", parent=None): """ :param shape: shape[0] = shape of previous layer c x h x w shape[1] = shape of current layer's weight f x c x h x w :param stride: stride :param padding: zero-padding :param parent: parent """ if parent is not None: _parent = parent.root if parent.is_sub_layer else parent shape, stride, padding = _parent.shape, _parent.stride, _parent.padding Layer.__init__(self, shape) self._stride = stride if isinstance(padding, str): if padding.upper() == "VALID": self._padding = 0 self._pad_flag = "VALID" else: self._padding = self._pad_flag = "SAME" else: self._padding = int(padding) self._pad_flag = "VALID" self.parent = parent if len(shape) == 1: self.n_channels = self.n_filters = self.out_h = self.out_w = None else: self.feed_shape(shape) def feed_shape(self, shape): self.shape = shape self.n_channels, height, width = shape[0] self.n_filters, filter_height, filter_width = shape[1] if self._pad_flag == "VALID": self.out_h = ceil((height - filter_height + 1) / self._stride) self.out_w = ceil((width - filter_width + 1) / self._stride) else: self.out_h = ceil(height / self._stride) self.out_w = ceil(width / self._stride) @property def stride(self): return self._stride @property def padding(self): return self._padding @property def pad_flag(self): return self._pad_flag
class TFSVM(KernelBase): TFSVMTiming = Timing() def __init__(self): super(TFSVM, self).__init__() self._fit_args, self._fit_args_names = [1e-3], ["tol"] self._cost = self._train_step = None self._sess = tf.Session() self._do_log = False def _prepare(self, sample_weight, **kwargs): lr = kwargs.get("lr", self._params["lr"]) sample_weight = tf.constant(sample_weight, dtype=tf.float32, name="sample_weight") x, y = tf.constant(self._x, dtype=tf.float32), tf.constant(self._y, dtype=tf.float32) self._gram = tf.constant(self._kernel(self._x, self._x), dtype=tf.float32, name="gram") self._w = tf.Variable(np.zeros(len(self._x))[None, ...], dtype=tf.float32, name="w") self._b = tf.Variable(.0, dtype=tf.float32, name="b") y_pred = self.predict(x, True, True) self._cost = tf.reduce_sum( tf.maximum(1 - y * y_pred, 0) * sample_weight) + 0.5 * tf.matmul( self._w, tf.matmul(self._gram, self._w, transpose_b=True))[0][0] self._train_step = tf.train.AdamOptimizer(learning_rate=lr).minimize( self._cost) self._sess.run(tf.global_variables_initializer()) @TFSVMTiming.timeit(level=1, prefix="[API] ") def _fit(self, sample_weight, tol): _l = self._sess.run([self._cost, self._train_step])[0] if _l < tol: return True @TFSVMTiming.timeit(level=1, prefix="[API] ") def predict(self, x, get_raw_results=False, gram_provided=False): if gram_provided: gram = self._gram else: gram = tf.constant(self._kernel(np.atleast_2d(x), self._x).T, dtype=tf.float32) rs = tf.matmul(self._w, gram) + self._b if not get_raw_results: rs = tf.sign(rs) if not gram_provided: rs = self._sess.run(rs) return rs
class TFSVM(TFKernelBase): TFSVMTiming = Timing() def __init__(self, **kwargs): super(TFSVM, self).__init__(**kwargs) self._fit_args, self._fit_args_names = [1e-3], ["tol"] self._batch_size = kwargs.get("batch_size", 128) self._optimizer = kwargs.get("optimizer", "Adam") self._train_repeat = None def _prepare(self, sample_weight, **kwargs): lr = kwargs.get("lr", self._params["lr"]) self._w = tf.Variable(np.zeros([len(self._x), 1]), dtype=tf.float32, name="w") self._b = tf.Variable(.0, dtype=tf.float32, name="b") self._tfx = tf.placeholder(tf.float32, [None, None]) self._tfy = tf.placeholder(tf.float32, [None]) self._y_pred_raw = tf.transpose( tf.matmul(self._tfx, self._w) + self._b) self._y_pred = tf.sign(self._y_pred_raw) self._loss = tf.reduce_sum( tf.maximum(1 - self._tfy * self._y_pred_raw, 0) * sample_weight) + 0.5 * tf.matmul( # self._w, tf.matmul(self._tfx, self._w, transpose_b=True) (self._y_pred_raw - self._b), self._w)[0][0] self._train_step = TFOptFac().get_optimizer_by_name( self._optimizer, lr).minimize(self._loss) self._sess.run(tf.global_variables_initializer()) @TFSVMTiming.timeit(level=1, prefix="[API] ") def _fit(self, sample_weight, tol): if self._train_repeat is None: self._train_repeat = self._get_train_repeat( self._x, self._batch_size) l = self._batch_training(self._gram, self._y, self._batch_size, self._train_repeat, self._loss, self._train_step) if l < tol: return True @TFSVMTiming.timeit(level=1, prefix="[API] ") def predict(self, x, get_raw_results=False, gram_provided=False): rs = self._y_pred_raw if get_raw_results else self._y_pred if gram_provided: return self._sess.run(rs, {self._tfx: x}).ravel() return self._sess.run( rs, { self._tfx: self._kernel(np.atleast_2d(x), self._x) }).ravel()
class TFLinearSVM(ClassifierBase): TFLinearSVMTiming = Timing() def __init__(self, **kwargs): super(TFLinearSVM, self).__init__(**kwargs) self._w = self._b = None self._sess = tf.Session() self._params["c"] = kwargs.get("c", 1) self._params["lr"] = kwargs.get("lr", 0.001) self._params["epoch"] = kwargs.get("epoch", 10 ** 4) self._params["tol"] = kwargs.get("tol", 1e-3) @TFLinearSVMTiming.timeit(level=1, prefix="[API] ") def fit(self, x, y, sample_weight=None, c=None, lr=None, epoch=None, tol=None): if sample_weight is None: sample_weight = self._params["sw"] if c is None: c = self._params["c"] if lr is None: lr = self._params["lr"] if epoch is None: epoch = self._params["epoch"] if tol is None: tol = self._params["tol"] if sample_weight is None: sample_weight = tf.constant(np.ones(len(y)), dtype=tf.float32, name="sample_weight") else: sample_weight = tf.constant(np.asarray(sample_weight) * len(y), dtype=tf.float32, name="sample_weight") x, y = tf.constant(x, dtype=tf.float32), tf.constant(y, dtype=tf.float32) self._w = tf.Variable(np.zeros(x.shape[1]), dtype=tf.float32, name="w") self._b = tf.Variable(0., dtype=tf.float32, name="b") y_pred = self.predict(x, True, False) cost = tf.reduce_sum(tf.maximum(1 - y * y_pred, 0) * sample_weight) + c * tf.nn.l2_loss(self._w) train_step = tf.train.AdamOptimizer(learning_rate=lr).minimize(cost) self._sess.run(tf.global_variables_initializer()) bar = ProgressBar(max_value=epoch, name="TFLinearSVM") for _ in range(epoch): _l = self._sess.run([cost, train_step])[0] if _l < tol: bar.update(epoch) break bar.update() @TFLinearSVMTiming.timeit(level=1, prefix="[API] ") def predict(self, x, get_raw_results=False, out_of_sess=True): rs = tf.reduce_sum(self._w * x, axis=1) + self._b if not get_raw_results: rs = tf.sign(rs) if out_of_sess: rs = self._sess.run(rs) return rs
class T1234(ClassifierBase, metaclass=ClassifierMeta): T1234Timing = Timing() @staticmethod @T1234Timing.timeit(level=1, prefix="[API] ") def predict(x): y_pred = np.zeros(len(x)) x_axis, y_axis = x.T x_lt_0, y_lt_0 = x_axis < 0, y_axis < 0 x_gt_0, y_gt_0 = ~x_lt_0, ~y_lt_0 y_pred[x_lt_0 & y_gt_0] = 1 y_pred[x_lt_0 & y_lt_0] = 2 y_pred[x_gt_0 & y_gt_0] = 3 return y_pred
class T1234(ClassifierBase): T1234Timing = Timing() @staticmethod @T1234Timing.timeit(level=1, prefix="[API] ") def predict(self, x, get_raw_results=False): y_pred = np.zeros(len(x)) x_axis, y_axis = x.T x_lt_0, y_lt_0 = x_axis < 0, y_axis < 0 x_gt_0, y_gt_0 = ~x_lt_0, ~y_lt_0 y_pred[x_lt_0 & y_gt_0] = 1 y_pred[x_lt_0 & y_lt_0] = 2 y_pred[x_gt_0 & y_gt_0] = 3 return y_pred
class NaiveBayes(ClassifierBase, metaclass=ClassifierMeta): NaiveBayesTiming = Timing() def __init__(self): self._x = self._y = None self._data = self._func = None self._n_possibilities = None self._labelled_x = self._label_zip = None self._cat_counter = self._con_counter = None self.label_dic = self._feat_dics = None def feed_data(self, x, y, sample_weight=None): pass def _feed_sample_weight(self, sample_weight=None): pass @NaiveBayesTiming.timeit(level=2, prefix="[API] ") def get_prior_probability(self, lb=1): return [(_c_num + lb) / (len(self._y) + lb * len(self._cat_counter)) for _c_num in self._cat_counter] @NaiveBayesTiming.timeit(level=2, prefix="[API] ") def fit(self, x=None, y=None, sample_weight=None, lb=1): if x is not None and y is not None: self.feed_data(x, y, sample_weight) self._func = self._fit(lb) def _fit(self, lb): pass @NaiveBayesTiming.timeit(level=1, prefix="[API] ") def predict(self, x, get_raw_result=False): if isinstance(x, np.ndarray): x = x.tolist() else: x = [xx[:] for xx in x] x = self._transfer_x(x) m_arg, m_probability = np.zeros(len(x), dtype=np.int8), np.zeros(len(x)) for i in range(len(self._cat_counter)): p = self._func(x, i) _mask = p > m_probability m_arg[_mask], m_probability[_mask] = i, p[_mask] if not get_raw_result: return np.array([self.label_dic[arg] for arg in m_arg]) return m_probability def _transfer_x(self, x): return x
class TFClassifierBase(ClassifierBase): """ Tensorfow classifier framework Implemented tensorflow ver. metrics """ clf_timing = Timing() def __init__(self, **kwargs): super(TFClassifierBase, self).__init__(**kwargs) self._tfx = self._tfy = None self._y_pred_raw = self._y_pred = None self._sess = tf.Session() @staticmethod @clf_timing.timeit(level=2, prefix="[Metric] ") def acc(y, y_pred, weights=None): y_arg, y_pred_arg = tf.argmax(y, axis=1), tf.argmax(y_pred, axis=1) same = tf.cast(tf.equal(y_arg, y_pred_arg), tf.float32) if weights is not None: same *= weights return tf.reduce_mean(same) @staticmethod @clf_timing.timeit(level=2, prefix="[Metric] ") def f1_score(y, y_pred): y_arg, y_pred_arg = tf.argmax(y, axis=1), tf.argmax(y_pred, axis=1) tp = tf.reduce_sum(y_arg * y_pred_arg) if tp == 0: return .0 fp = tf.reduce_sum((1 - y_arg) * y_pred_arg) fn = tf.reduce_sum(y_arg * (1 - y_pred_arg)) return 2 * tp / (2 * tp + fn + fp) @clf_timing.timeit(level=2, prefix="[Core] ") def _batch_training(self, x, y, batch_size, train_repeat, *args): loss, train_step, *args = args epoch_cost = 0 for i in range(train_repeat): if train_repeat != 1: batch = np.random.choice(len(x), batch_size) x_batch, y_batch = x[batch], y[batch] else: x_batch, y_batch = x, y epoch_cost += self._sess.run([loss, train_step], { self._tfx: x_batch, self._tfy: y_batch })[0] self._batch_work(i, *args) return epoch_cost / train_repeat
def vis_test(): nn = NNDist() epoch = 1000 record_period = 4 make_mp4 = True timing = Timing(enabled=True) timing_level = 1 x, y = DataUtil.gen_spiral(50, 3, 3, 2.5) nn.build([x.shape[1], 6, 6, 6, y.shape[1]]) nn.optimizer = "Adam" nn.preview() nn.feed_timing(timing) nn.fit(x, y, verbose=1, record_period=record_period, epoch=epoch, train_only=True, draw_detailed_network=True, make_mp4=make_mp4, show_animation=True) nn.draw_results() timing.show_timing_log(timing_level)
class GDKP(GDKernelBase): GDKPTiming = Timing() @GDKPTiming.timeit(level=1, prefix="[Core] ") def _get_grads(self, x_batch, y_batch, y_pred, sample_weight_batch, *args): err = -y_batch * (x_batch.dot(self._alpha) + self._b) * sample_weight_batch mask = err >= 0 # type: np.ndarray if not np.any(mask): self._model_grads = [None, None] else: delta = -y_batch[mask] * sample_weight_batch[mask] self._model_grads = [ np.sum(delta[..., None] * x_batch[mask], axis=0), np.sum(delta) ] return np.sum(err[mask])
class GDSVM(GDKernelBase): GDSVMTiming = Timing() @GDSVMTiming.timeit(level=1, prefix="[Core] ") def _get_grads(self, x_batch, y_batch, y_pred, sample_weight_batch, *args): err = -y_batch * (x_batch.dot(self._alpha) + self._b) mask = err >= 0 if np.max(err) < 0: self._model_grads = [None, None] else: delta = -y_batch[mask] * sample_weight_batch[mask] self._model_grads = [ np.sum(delta[..., None] * x_batch[mask], axis=0), np.sum(delta) ] return np.sum(err[mask]) + 0.5 * (y_pred - self._b).dot(self._alpha)
class ConvLayer(Layer): LayerTiming = Timing() def __init__(self, shape, stride=1, padding=None, parent=None): """ :param shape: shape[0] = shape of previous layer c x h x w shape[1] = shape of current layer's weight f x h x w :param stride: stride :param padding: zero-padding :param parent: parent """ if parent is not None: _parent = parent.root if parent.is_sub_layer else parent shape = _parent.shape Layer.__init__(self, shape) self.stride = stride if padding is None: padding = "SAME" if isinstance(padding, str): if padding.upper() == "VALID": self.padding = 0 self.pad_flag = "VALID" else: self.padding = self.pad_flag = "SAME" elif isinstance(padding, int): self.padding = padding self.pad_flag = "VALID" else: raise ValueError("Padding should be 'SAME' or 'VALID' or integer") self.parent = parent if len(shape) == 1: self.n_channels = self.n_filters = self.out_h = self.out_w = None else: self.feed_shape(shape) def feed_shape(self, shape): self.shape = shape self.n_channels, height, width = shape[0] self.n_filters, filter_height, filter_width = shape[1] if self.pad_flag == "VALID": self.out_h = ceil((height - filter_height + 1) / self.stride) self.out_w = ceil((width - filter_width + 1) / self.stride) else: self.out_h = ceil(height / self.stride) self.out_w = ceil(width / self.stride)
class GDKernelBase(KernelBase, GDBase): """ Kernel classifier with gradient descent algorithm """ GDKernelBaseTiming = Timing() def __init__(self, **kwargs): super(GDKernelBase, self).__init__(**kwargs) self._fit_args, self._fit_args_names = [1e-3], ["tol"] self._batch_size = kwargs.get("batch_size", 128) self._optimizer = kwargs.get("optimizer", "Adam") self._train_repeat = 0 def _prepare(self, sample_weight, **kwargs): lr = kwargs.get("lr", self._params["lr"]) self._alpha = np.random.random(len(self._x)).astype(np.float32) self._b = np.random.random(1).astype(np.float32) self._model_parameters = [self._alpha, self._b] self._optimizer = OptFactory().get_optimizer_by_name( self._optimizer, self._model_parameters, lr, self._params["epoch"]) @GDKernelBaseTiming.timeit(level=1, prefix="[Core] ") def _fit(self, sample_weight, tol): if self._train_repeat == 0: self._train_repeat = self._get_train_repeat( self._x, self._batch_size) l = self._batch_training(self._gram, self._y, self._batch_size, self._train_repeat, sample_weight, gram_provided=True) if l < tol: return True @GDKernelBaseTiming.timeit(level=1, prefix="[API] ") def predict(self, x, get_raw_results=False, gram_provided=False): if not gram_provided: x = self._kernel(self._x, np.atleast_2d(x)) elif self._alpha.shape[0] != x.shape[0]: x = x.T y_pred = (self._alpha.dot(x) + self._b).ravel() if not get_raw_results: return np.sign(y_pred) return y_pred