def predict(self, x): if self.learned is False: raise Exception('this tree has not been fitted') x = check_feature_input(x) x = Normalizer.normalize(x, norm=self.normalize, axis=self.axis) samples, features = x.shape if features != self.features: raise Exception('inconsistent attribute number') results = [] for index in range(samples): values = x[index] treenode = self.root # traverse the tree until reach the end while not treenode.is_leaf: attr_index, split_value = treenode.split_info if values[attr_index] <= split_value: treenode = treenode.left else: treenode = treenode.right results.append(treenode.get_prediected_labels()) return results
def initialize(self, x, y): x = check_feature_input(x) y = check_target_input(y) self.samples, self.features = x.shape self.classes = y.shape[1] self.all_attributes = x self.bin_labels = y self.instances = np.array([i for i in range(self.samples)]) # pure tests: check if all the samples belong to same label set y_list = y.tolist() self.pure = y_list.count(y_list[0]) == len(y_list)
def fit(self, x, y): """Fit underlying estimators. Parameters ---------- x : (sparse) array-like, shape = [n_samples, n_features] Data. y : (sparse) array-like, shape = [n_samples, ], [n_samples, n_classes] Multi-Label targets. Returns ------- self """ # if self.trained is True: # raise Exception('this classifier has already been trained, please create a new classifier') x = check_feature_input(x) y = check_target_input(y) self.features = x.shape[1] self.classes = y.shape[1] self.dataset = self.prepare_data(x, y) self.samples = len(self.dataset) if self.samples < self.features: raise Exception("Your must have more instances than features") self.neural_num = int(self.features * self.neural_percent) # weights between the hidden layer and the output layer self.wsj_matrix = np.random.random_sample( (self.neural_num, self.classes)) - 0.5 # weights between the input layer and the hidden layer self.vhs_matrix = np.random.random_sample( (self.features, self.neural_num)) - 0.5 # bias between the hidden layer and the output layer self.bias_b = np.ones((1, self.classes)) # bias between the input layer and the hidden layer self.bias_a = np.ones((1, self.neural_num)) self.iterate_training() self.trained = True return self
def fit(self, x, y): """Fit underlying estimators. Parameters ---------- x : (sparse) array-like, shape = [n_samples, n_features] Data. y : (sparse) array-like, shape = [n_samples, ], [n_samples, n_classes] Multi-Label targets. Returns ------- self """ # if self.trained is True: # raise Exception('this classifier has already been trained, please create a new classifier') x = check_feature_input(x) y = check_target_input(y) self.features = x.shape[1] self.classes = y.shape[1] self.dataset = self.prepare_data(x, y) self.samples = len(self.dataset) if self.samples < self.features: raise Exception("Your must have more instances than features") self.neural_num = int(self.features * self.neural_percent) # weights between the hidden layer and the output layer self.wsj_matrix = np.random.random_sample((self.neural_num, self.classes)) - 0.5 # weights between the input layer and the hidden layer self.vhs_matrix = np.random.random_sample((self.features, self.neural_num)) - 0.5 # bias between the hidden layer and the output layer self.bias_b = np.ones((1, self.classes)) # bias between the input layer and the hidden layer self.bias_a = np.ones((1, self.neural_num)) self.iterate_training() self.trained = True return self
def predict(self, x, rank_results=False): """ predict process :param x: feature matrix :param rank_results: decide whether return RankResults object or the actual output :return: result: RankResults """ if self.trained is False: raise Exception('this classifier has not been trained') x = check_feature_input(x) samples, features = x.shape if features != self.features: raise Exception("inconsistent feature dimension") x = Normalizer.normalize(x, norm=self.normalize, axis=self.axis) result = RankResults() for sample_index in range(samples): sample_result = [] c = self.forward_propagation(x[sample_index])[1][0] threshold = self.threshold.compute_threshold(c) top_label = None max_value = 0 count = 0 for j in range(self.classes): if c[j] >= threshold: count += 1 sample_result.append(j) if c[j] > max_value: top_label = j max_value = c[j] # append the top label if no label satisfies the threshold value if count == 0: sample_result.append(top_label) result.add(sample_result, top_label, c) if rank_results is False: result = result.predictedLabels return result
def predict(self, x, rank_results=False): if self.trained is False: raise Exception('this classifier has not been trained') x = check_feature_input(x) x = Normalizer.normalize(x, self.normalize, self.axis) sample_num, feature_num = x.shape class_num = self.w.shape[0] if feature_num != self.w.shape[1] - 1: raise Exception( 'testing samples have inconsistent shape of training samples!') x_extend = np.concatenate((x, np.array([np.ones(sample_num)]).T), axis=1) threshold = self.threshold outputs = np.dot(x_extend, self.w.T) result = RankResults() for index in range(sample_num): sample_result = [] op = outputs[index] th = threshold.compute_threshold(op) top_label = None max_value = float('-inf') count = 0 for j in range(class_num): if op[j] >= th: count += 1 sample_result.append(j) if op[j] > max_value: top_label = j max_value = op[j] if count == 0: sample_result.append(top_label) result.add(sample_result, top_label, op) if rank_results is False: result = result.predictedLabels return result
def predict(self, x, rank_results=False): if self.trained is False: raise Exception('this classifier has not been trained') x = check_feature_input(x) x = Normalizer.normalize(x, self.normalize, self.axis) sample_num, feature_num = x.shape class_num = self.w.shape[0] if feature_num != self.w.shape[1] - 1: raise Exception('testing samples have inconsistent shape of training samples!') x_extend = np.concatenate((x, np.array([np.ones(sample_num)]).T), axis=1) threshold = self.threshold outputs = np.dot(x_extend, self.w.T) result = RankResults() for index in range(sample_num): sample_result = [] op = outputs[index] th = threshold.compute_threshold(op) top_label = None max_value = float('-inf') count = 0 for j in range(class_num): if op[j] >= th: count += 1 sample_result.append(j) if op[j] > max_value: top_label = j max_value = op[j] if count == 0: sample_result.append(top_label) result.add(sample_result, top_label, op) if rank_results is False: result = result.predictedLabels return result
def fit(self, x, y, c_factor): x = check_feature_input(x) y = check_target_input(y) self.features = x.shape[1] x = Normalizer.normalize(x, self.normalize, self.axis) sample_num, feature_num = x.shape class_num = y.shape[1] class_info = AllLabelInfo() """ Franke and Wolfe Method applied on the optimization problem """ # organize labels for preparation for sample_index in range(sample_num): sample_y = y[sample_index] labels = [] not_labels = [] for label_index in range(class_num): if sample_y[label_index] == 1: labels.append(label_index) else: not_labels.append(label_index) class_info.append(labels, not_labels) """ Alpha ought to have 3-dimensions. For convenience, it is flattened into a list. It's length is sum(yi*nyi) for i in range(samle_num) """ alpha = np.zeros(class_info.totalProduct) alpha_var = cvx.Variable(class_info.totalProduct) """ C has 4 dimensions, which is hard to present. In this program it has 2 dimensions, which is i(sample index) and k(class index). Each c[i][k] contains a yi*nyi array, which is initialized according to the original paper. """ c = [[0 for k in range(class_num)] for i in range(sample_num)] for i in range(sample_num): sample_shape, labels, not_labels = class_info.get_shape(i, True) for k in range(class_num): matrix = np.zeros(sample_shape) if k in labels: index = labels.index(k) matrix[index, :] = 1 else: index = not_labels.index(k) matrix[:, index] = -1 c[i][k] = matrix.flatten() c = np.array(c) beta = np.zeros((class_num, sample_num)) beta_new = np.zeros((class_num, sample_num)) wx_inner = np.zeros((class_num, sample_num)) # TODO: this can cut half of the running time x_inner = np.array([[np.inner(x[i], x[j]) for j in range(sample_num)] for i in range(sample_num)]) g_ikl = np.zeros(class_info.totalProduct) """ prepare for the first linear programming """ c_i = class_info.eachProduct bnds = [] for i in range(sample_num): bnds += [c_factor / c_i[i] for j in range(c_i[i])] bnds = np.array(bnds) zeros = np.zeros(class_info.totalProduct) zeros.fill(1e-5) A_lp = [] for k in range(1, class_num): A_lp.append(np.concatenate(c[:, k]).tolist()) A_lp = np.array(A_lp) b_lp = np.zeros(class_num - 1) cons = [ zeros <= alpha_var, alpha_var <= bnds, A_lp * alpha_var == b_lp ] converge = False iteration_count = 0 # iterate training until converge while not converge: iteration_count += 1 # compute beta for i in range(sample_num): alpha_range = class_info.get_range(i) alpha_piece = alpha[alpha_range[0]:alpha_range[1]] c_list = c[i] for k in range(class_num): beta[k][i] = np.inner(c_list[k], alpha_piece) # compute <w_k, x_j> for k in range(class_num): beta_list = beta[k] for j in range(sample_num): x_innerlist = x_inner[:, j] wx_inner[k][j] = np.inner(beta_list, x_innerlist) # compute g_ikl for i in range(sample_num): g_range = class_info.get_range(i) shape, labels, not_labels = class_info.get_shape(i, True) wx_list = wx_inner[:, i] g_ikl[g_range[0]:g_range[1]] = np.repeat( wx_list[labels], shape[1]) - np.tile( wx_list[not_labels], shape[0]) - 1 """ optimization problem 1: solve min<g, alpha_new> with corresponding constraints """ if self.print_procedure: print('iteration %d...' % iteration_count) """ cvxopt.solvers.lp Solves a pair of primal and dual LPs minimize c'*x subject to G*x + s = h A*x = b s >= 0 maximize -h'*z - b'*y subject to G'*z + A'*y + c = 0 z >= 0. cvxopt.solvers.lp(c, G, h[, A, b[, solver[, primalstart[, dualstart]]]]) """ obj = cvx.Minimize(cvx.sum_entries(g_ikl * alpha_var)) prob = cvx.Problem(obj, cons) prob.solve() alpha_new = np.array(alpha_var.value).T[0] """ now the problem collapse into a really simple qp problem """ # compute beta_new for i in range(sample_num): alpha_range = class_info.get_range(i) alpha_piece = alpha_new[alpha_range[0]:alpha_range[1]] c_list = c[i] for k in range(class_num): beta_new[k][i] = np.inner(c_list[k], alpha_piece) """ We need to find lambda which will make W(alpha + lambda*alpha_new) be maximum and alpha + lambda*alpha_new satisfies the previous constraints. After calculating the formula, it is now: new_W = old_W + [sum formula of beta_new and beta] + [sum formula of beta and beta_new] + [sum formula of beta_new and beta_new] + [sum of the new alpha] old_W is fixed and has no effect on the choice of the lambda during the maximum process. Then we can calculate the coeffient of lambda. The final problem will look like: maximize a*lambda_square + b*lambda c <= lambda <= d It is apparently easy to solve. """ # init coeffient of lambda lambda_11 = np.sum(beta_new.T.dot(beta) * x_inner) lambda_12 = np.sum(beta.T.dot(beta_new) * x_inner) lambda_13 = np.sum(alpha_new) # coefficient of lambda lambda_1 = lambda_13 - lambda_11 / 2 - lambda_12 / 2 # coefficient of lambda square lambda_2 = np.sum(beta_new.T.dot(beta_new) * x_inner) / (-2) # prepare constraints left_vec = -alpha right_vec = bnds - alpha left = float('-inf') right = float('inf') for alpha_index in range(class_info.totalProduct): if not alpha_new[alpha_index] == 0: left = max(left_vec[alpha_index] / alpha_new[alpha_index], left) right = min( right_vec[alpha_index] / alpha_new[alpha_index], right) optifunc = lambda z: lambda_2 * z * z + lambda_1 * z # decide lambda's value if lambda_2 < 0: opti_lambda = -lambda_1 / (lambda_2 * 2) if opti_lambda < left: final_lambda = left elif opti_lambda > right: final_lambda = right else: final_lambda = opti_lambda elif lambda_2 == 0: if lambda_1 >= 0: final_lambda = right else: final_lambda = left else: worst_lambda = -lambda_1 / (lambda_2 * 2) if worst_lambda < left: final_lambda = right elif worst_lambda > right: final_lambda = left else: final_lambda = left if optifunc(left) >= optifunc( right) else right if self.print_procedure: print("final lambda: " + str(final_lambda)) print("optifunc: " + str(optifunc(final_lambda))) # converge condition if optifunc(final_lambda) <= 1 or final_lambda <= 1e-3: converge = True else: alpha += final_lambda * alpha_new """ compute w and b via KKT conditions """ w = [0 for i in range(class_num)] for k in range(class_num): beta_vec = np.asarray([beta[k]]) w[k] = beta_vec.dot(x)[0] w = np.array(w) b = np.zeros(class_num) # use x[0] to compute differences of b x_list = x[0] shape, labels, not_labels = class_info.get_shape(0, True) """ We know that once the differences between each element and the first element is known we can get all the values of the list. As the classification system is based on ranking, we can make any element 0 as a start of calculation, which will not affect the final ranking. """ # make the first label's b=0, it won't affect the fianl ranking for l in not_labels: b[l] = np.inner(w[labels[0]] - w[l], x_list) - 1 # then use b[falselabels[0]] to compute b[actuallabels[1:]] falselabelb = b[not_labels[0]] falselabel_index = not_labels[0] for labelIndex in range(1, len(labels)): b[labels[labelIndex]] = 1 + falselabelb - np.inner( w[labels[labelIndex]] - w[falselabel_index], x_list) # build threshold for labeling x_extend = np.concatenate((x, np.array([np.ones(sample_num)]).T), axis=1) w_extend = np.concatenate((w, np.array([b]).T), axis=1) model_outputs = np.dot(x_extend, w_extend.T) self.threshold = ThresholdFunction(model_outputs, y) self.w = w_extend self.trained = True return self
def fit(self, x, y, c_factor): x = check_feature_input(x) y = check_target_input(y) self.features = x.shape[1] x = Normalizer.normalize(x, self.normalize, self.axis) sample_num, feature_num = x.shape class_num = y.shape[1] class_info = AllLabelInfo() """ Franke and Wolfe Method applied on the optimization problem """ # organize labels for preparation for sample_index in range(sample_num): sample_y = y[sample_index] labels = [] not_labels = [] for label_index in range(class_num): if sample_y[label_index] == 1: labels.append(label_index) else: not_labels.append(label_index) class_info.append(labels, not_labels) """ Alpha ought to have 3-dimensions. For convenience, it is flattened into a list. It's length is sum(yi*nyi) for i in range(samle_num) """ alpha = np.zeros(class_info.totalProduct) alpha_var = cvx.Variable(class_info.totalProduct) """ C has 4 dimensions, which is hard to present. In this program it has 2 dimensions, which is i(sample index) and k(class index). Each c[i][k] contains a yi*nyi array, which is initialized according to the original paper. """ c = [[0 for k in range(class_num)] for i in range(sample_num)] for i in range(sample_num): sample_shape, labels, not_labels = class_info.get_shape(i, True) for k in range(class_num): matrix = np.zeros(sample_shape) if k in labels: index = labels.index(k) matrix[index, :] = 1 else: index = not_labels.index(k) matrix[:, index] = -1 c[i][k] = matrix.flatten() c = np.array(c) beta = np.zeros((class_num, sample_num)) beta_new = np.zeros((class_num, sample_num)) wx_inner = np.zeros((class_num, sample_num)) # TODO: this can cut half of the running time x_inner = np.array([[np.inner(x[i], x[j]) for j in range(sample_num)] for i in range(sample_num)]) g_ikl = np.zeros(class_info.totalProduct) """ prepare for the first linear programming """ c_i = class_info.eachProduct bnds = [] for i in range(sample_num): bnds += [c_factor / c_i[i] for j in range(c_i[i])] bnds = np.array(bnds) zeros = np.zeros(class_info.totalProduct) zeros.fill(1e-5) A_lp = [] for k in range(1, class_num): A_lp.append(np.concatenate(c[:, k]).tolist()) A_lp = np.array(A_lp) b_lp = np.zeros(class_num - 1) cons = [zeros <= alpha_var, alpha_var <= bnds, A_lp * alpha_var == b_lp] converge = False iteration_count = 0 # iterate training until converge while not converge: iteration_count += 1 # compute beta for i in range(sample_num): alpha_range = class_info.get_range(i) alpha_piece = alpha[alpha_range[0]:alpha_range[1]] c_list = c[i] for k in range(class_num): beta[k][i] = np.inner(c_list[k], alpha_piece) # compute <w_k, x_j> for k in range(class_num): beta_list = beta[k] for j in range(sample_num): x_innerlist = x_inner[:, j] wx_inner[k][j] = np.inner(beta_list, x_innerlist) # compute g_ikl for i in range(sample_num): g_range = class_info.get_range(i) shape, labels, not_labels = class_info.get_shape(i, True) wx_list = wx_inner[:, i] g_ikl[g_range[0]:g_range[1]] = np.repeat(wx_list[labels], shape[1]) - np.tile(wx_list[not_labels], shape[0]) - 1 """ optimization problem 1: solve min<g, alpha_new> with corresponding constraints """ if self.print_procedure: print('iteration %d...' % iteration_count) """ cvxopt.solvers.lp Solves a pair of primal and dual LPs minimize c'*x subject to G*x + s = h A*x = b s >= 0 maximize -h'*z - b'*y subject to G'*z + A'*y + c = 0 z >= 0. cvxopt.solvers.lp(c, G, h[, A, b[, solver[, primalstart[, dualstart]]]]) """ obj = cvx.Minimize(cvx.sum_entries(g_ikl * alpha_var)) prob = cvx.Problem(obj, cons) prob.solve() alpha_new = np.array(alpha_var.value).T[0] """ now the problem collapse into a really simple qp problem """ # compute beta_new for i in range(sample_num): alpha_range = class_info.get_range(i) alpha_piece = alpha_new[alpha_range[0]:alpha_range[1]] c_list = c[i] for k in range(class_num): beta_new[k][i] = np.inner(c_list[k], alpha_piece) """ We need to find lambda which will make W(alpha + lambda*alpha_new) be maximum and alpha + lambda*alpha_new satisfies the previous constraints. After calculating the formula, it is now: new_W = old_W + [sum formula of beta_new and beta] + [sum formula of beta and beta_new] + [sum formula of beta_new and beta_new] + [sum of the new alpha] old_W is fixed and has no effect on the choice of the lambda during the maximum process. Then we can calculate the coeffient of lambda. The final problem will look like: maximize a*lambda_square + b*lambda c <= lambda <= d It is apparently easy to solve. """ # init coeffient of lambda lambda_11 = np.sum(beta_new.T.dot(beta) * x_inner) lambda_12 = np.sum(beta.T.dot(beta_new) * x_inner) lambda_13 = np.sum(alpha_new) # coefficient of lambda lambda_1 = lambda_13 - lambda_11 / 2 - lambda_12 / 2 # coefficient of lambda square lambda_2 = np.sum(beta_new.T.dot(beta_new) * x_inner) / (-2) # prepare constraints left_vec = - alpha right_vec = bnds - alpha left = float('-inf') right = float('inf') for alpha_index in range(class_info.totalProduct): if not alpha_new[alpha_index] == 0: left = max(left_vec[alpha_index] / alpha_new[alpha_index], left) right = min(right_vec[alpha_index] / alpha_new[alpha_index], right) optifunc = lambda z: lambda_2 * z * z + lambda_1 * z # decide lambda's value if lambda_2 < 0: opti_lambda = -lambda_1 / (lambda_2 * 2) if opti_lambda < left: final_lambda = left elif opti_lambda > right: final_lambda = right else: final_lambda = opti_lambda elif lambda_2 == 0: if lambda_1 >= 0: final_lambda = right else: final_lambda = left else: worst_lambda = -lambda_1 / (lambda_2 * 2) if worst_lambda < left: final_lambda = right elif worst_lambda > right: final_lambda = left else: final_lambda = left if optifunc(left) >= optifunc(right) else right if self.print_procedure: print("final lambda: " + str(final_lambda)) print("optifunc: " + str(optifunc(final_lambda))) # converge condition if optifunc(final_lambda) <= 1 or final_lambda <= 1e-3: converge = True else: alpha += final_lambda * alpha_new """ compute w and b via KKT conditions """ w = [0 for i in range(class_num)] for k in range(class_num): beta_vec = np.asarray([beta[k]]) w[k] = beta_vec.dot(x)[0] w = np.array(w) b = np.zeros(class_num) # use x[0] to compute differences of b x_list = x[0] shape, labels, not_labels = class_info.get_shape(0, True) """ We know that once the differences between each element and the first element is known we can get all the values of the list. As the classification system is based on ranking, we can make any element 0 as a start of calculation, which will not affect the final ranking. """ # make the first label's b=0, it won't affect the fianl ranking for l in not_labels: b[l] = np.inner(w[labels[0]] - w[l], x_list) - 1 # then use b[falselabels[0]] to compute b[actuallabels[1:]] falselabelb = b[not_labels[0]] falselabel_index = not_labels[0] for labelIndex in range(1, len(labels)): b[labels[labelIndex]] = 1 + falselabelb - np.inner(w[labels[labelIndex]] - w[falselabel_index], x_list) # build threshold for labeling x_extend = np.concatenate((x, np.array([np.ones(sample_num)]).T), axis=1) w_extend = np.concatenate((w, np.array([b]).T), axis=1) model_outputs = np.dot(x_extend, w_extend.T) self.threshold = ThresholdFunction(model_outputs, y) self.w = w_extend self.trained = True return self