def create_member(self, data_files): #Gets the training indexes if self.member_number > 0: train_indexes = \ self.resampler.make_new_train(self.params.resample_size) else: train_indexes = [None, None] #Packs the needed data dataset = [train_indexes, data_files] #Trains the model m = mlp.sequential_model(dataset, self.params, member_number=self.member_number) #Gets the errors for the train set and updates the weights print('Getting the train errors and updating the weights') errors = common.errors(m, data_files[0], self.params.batch_size) e = np.sum((errors * self.D)) if e > 0: n_classes = data_files[0]['y'].shape[1] alpha = .5 * (math.log((1 - e) / e) + math.log(n_classes - 1)) if alpha <= 0.0: #By setting to 0 (instead of crashing), we should avoid # cicleci problems print("\nWARNING - NEGATIVE ALPHA (setting to 0.0)\n") alpha = 0.0 w = np.where(errors == 1, self.D * math.exp(alpha), self.D * math.exp(-alpha)) self.D = w / w.sum() else: alpha = 1.0 / (self.member_number + 1) self.resampler.update_weights(self.D) self.alphas.append(alpha) self.member_number += 1 return (m.to_yaml(), m.get_weights())
def create_member(self): train_set, sample_weights = self.resampler.make_new_train( self.params.resample_size) if self.member_number > 0: resampled = [ train_set, self.resampler.get_valid(), self.resampler.get_test() ] else: sample_weights = None resampled = [ self.resampler.get_train(), self.resampler.get_valid(), self.resampler.get_test() ] m = mlp.sequential_model(resampled, self.params, member_number=self.member_number, sample_weight=sample_weights) orig_train = self.resampler.get_train() errors = common.errors(m, orig_train[0], orig_train[1]) e = np.sum((errors * self.D)) if e > 0: alpha = .5 * math.log((1 - e) / e) w = np.where(errors == 1, self.D * math.exp(alpha), self.D * math.exp(-alpha)) self.D = w / w.sum() else: alpha = 1.0 / (self.member_number + 1) self.resampler.update_weights(self.D) self.alphas.append(alpha) self.member_number += 1 return (m.to_yaml(), m.get_weights())
def create_member(self): train_set, sample_weights = self.resampler.make_new_train( self.params.resample_size) if self.member_number > 0: resampled = [ train_set, self.resampler.get_valid(), self.resampler.get_test() ] else: resampled = [ self.resampler.get_train(), self.resampler.get_valid(), self.resampler.get_test() ] m = mlp.sequential_model( resampled, self.params, # <--- trains the new model(step 2 in [1]) member_number=self.member_number) orig_train = self.resampler.get_train() distance = common.distance( m, orig_train[0], orig_train[1]) # <--- loss for each element (steps 3 and 4 in [1]) max_dist = distance.max() distance_norm = distance / max_dist # <--- the loss function is now normalized in range [0,1] weighted_dist = np.sum( (distance_norm * self.D)) # <--- average weighted loss (step 5 in [1]) beta = weighted_dist / ( 1 - weighted_dist ) # <--- computation of the confidence in the predictor (step 6 in [1]) # [low beta = good prediction] w = self.D * ( beta**(1 - distance_norm) ) # <--- updates the weights for each sample (step 7 in [1]) self.D = w / w.sum() #alpha = the better the model is (smaller beta), the bigger alpha will be [alpha is computed to maintain consistency with other models] alpha = 0.5 * math.log(1 / beta) self.resampler.update_weights(self.D) self.alphas.append(alpha) self.member_number += 1 return (m.to_yaml(), m.get_weights())
def create_member(self, data_files): #gets the training indexes if self.member_number > 0: train_indexes = self.resampler.make_new_train( self.params.resample_size) else: train_indexes = [None, None] #packs the needed data dataset = [train_indexes, data_files] #trains the model m = mlp.sequential_model(dataset, self.params, member_number=self.member_number) self.member_number += 1 return (m.to_yaml(), m.get_weights())
def create_member(self): self.set_defaults() if self.member_number > 0: train_set, sample_weights = self.resampler.make_new_train( self.params.resample_size) resampled = [ train_set, self.resampler.get_valid(), self.resampler.get_test() ] else: resampled = [ self.resampler.get_train(), self.resampler.get_valid(), self.resampler.get_test() ] sample_weights = None if self.member_number > 0: self.params.n_epochs = self.n_epochs_after_first m = mlp.sequential_model( resampled, self.params, member_number=self.member_number, model_weights=self.weights, #the copy is because there is a bug in Keras that deletes names model_config=copy.deepcopy(self.model_config), frozen_layers=self.frozen_layers) self.weights = [l.get_weights() for l in m.layers] injection_index = self.incremental_index + self.member_number if self.incremental_layers is not None: if injection_index == -1: injection_index = len(self.model_config) new_layers = [] for i, l in enumerate(self.incremental_layers): new_layers.append(copy.deepcopy(l)) #make residual block new_block = self._residual_block(injection_index, new_layers, m, self.member_number) new_model_config = self.model_config[:injection_index] + [ new_block ] + self.model_config[injection_index:] if self.freeze_old_layers: self.frozen_layers = list(range(0, injection_index)) self.model_config = copy.deepcopy(new_model_config) self.weights = self.weights[:injection_index] self.member_number += 1 return (m.to_yaml(), m.get_weights())
def create_member(self): train_set, sample_weights = self.resampler.make_new_train( self.params.resample_size) if self.member_number > 0: resampled = [ train_set, self.resampler.get_valid(), self.resampler.get_test() ] else: resampled = [ self.resampler.get_train(), self.resampler.get_valid(), self.resampler.get_test() ] m = mlp.sequential_model(resampled, self.params, member_number=self.member_number) self.member_number += 1 return (m.to_yaml(), m.get_weights())
def create_member(self, data_files): #Gets the training indexes and defines c, if needed if self.member_number > 0: train_indexes = \ self.resampler.make_new_train(self.params.resample_size) else: train_indexes = [None, None] sample_counts = common.count_classes(data_files[0]) self.c = np.sum(np.square(sample_counts / self.train_size)) #Packs the needed data dataset = [train_indexes, data_files] #Trains the model m = mlp.sequential_model(dataset, self.params, member_number=self.member_number) #Gets the errors for the train set and updates the weights print('Getting the confidence and updating the weights') h = common.confidence(m, data_files[0], self.params.batch_size) r = np.sum((h * self.D)) if r > self.c: alpha = math.log(((1 - self.c) * r) / (self.c * (1 - r))) if alpha <= 0.0: #By setting to 0 (instead of crashing), we should avoid # cicleci problems print("\nWARNING - NEGATIVE ALPHA (setting to 0.0)\n") alpha = 0.0 w = self.D * math.exp(-alpha * (h - self.c)) self.D = w / w.sum() else: #This model should be discarded, since it's worse than containing # no additional information alpha = 0.0 self.resampler.update_weights(self.D) self.alphas.append(alpha) self.member_number += 1 return (m.to_yaml(), m.get_weights())
def create_member(self): self.set_defaults() if self.member_number > 0: if self.resample: train_set, sample_weights = self.resampler.make_new_train( self.params.resample_size) resampled = [ train_set, self.resampler.get_valid(), self.resampler.get_test() ] else: sample_weights = self.D resampled = [ self.resampler.get_train(), self.resampler.get_valid(), self.resampler.get_test() ] else: sample_weights = None resampled = [ self.resampler.get_train(), self.resampler.get_valid(), self.resampler.get_test() ] if self.member_number > 0: self.params.n_epochs = self.n_epochs_after_first if not self.use_sample_weights: sample_weights = None m = mlp.sequential_model( resampled, self.params, member_number=self.member_number, model_weights=self.weights, #the copy is because there is a bug in Keras that deletes names model_config=copy.deepcopy(self.model_config), frozen_layers=self.frozen_layers, sample_weight=sample_weights) self.weights = [l.get_weights() for l in m.layers] injection_index = self.incremental_index + self.member_number if self.incremental_layers is not None: if injection_index == -1: injection_index = len(self.model_config) new_layers = [] for i, l in enumerate(self.incremental_layers): new_layers.append(copy.deepcopy(l)) #make residual block new_block = self._residual_block(injection_index, new_layers, m, self.member_number) new_model_config = self.model_config[:injection_index] + [ new_block ] + self.model_config[injection_index:] if self.freeze_old_layers: self.frozen_layers = list(range(0, injection_index)) self.model_config = copy.deepcopy(new_model_config) self.weights = self.weights[:injection_index] orig_train = self.resampler.get_train() K = orig_train[1].shape[1] self.n_classes = K errors = common.errors(m, orig_train[0], orig_train[1]) error_rate = np.mean(errors) if error_rate >= 1. - (1. / K): return (None, None, False) if self.real: #Real BRN print(("-" * 40)) print(("error rate: {}".format(error_rate))) if error_rate > 0: continue_boosting = True y_coding = np.where(orig_train[1] == 0., -1. / (K - 1), 1.) proba = m.predict(orig_train[0]) proba[proba < np.finfo(proba.dtype).eps] = np.finfo( proba.dtype).eps print((proba[:10])) print((self.D[:10])) factor = np.exp( -1. * (((K - 1.) / K) * inner1d(y_coding, np.log(proba)))) print((factor[:10])) w = self.D * factor print((w[:10])) self.D = w / w.sum() self.resampler.update_weights(self.D) else: continue_boosting = not self.early_stopping self.member_number += 1 return (m.to_yaml(), m.get_weights(), continue_boosting) else: if error_rate > 0: continue_boosting = True #e = sum((errors * self.D)) / sum(self.D) e = np.average(errors, weights=self.D) alpha = math.log((1 - e) / e) + math.log(K - 1) factor = np.clip( np.where(errors == 1, math.exp(alpha), math.exp(-alpha)), 1e-3, 1e3) w = self.D * factor self.D = w / w.sum() self.resampler.update_weights(self.D) else: continue_boosting = not self.early_stopping alpha = 1. / (self.member_number + 1) self.alphas.append(alpha) self.member_number += 1 return (m.to_yaml(), m.get_weights(), continue_boosting)
def create_member(self): self.set_defaults() train_set, sample_weights = self.resampler.make_new_train( self.params.resample_size) if self.member_number > 0: if self.resample: resampled = [ train_set, self.resampler.get_valid(), self.resampler.get_test() ] else: resampled = [ self.resampler.get_train(), self.resampler.get_valid(), self.resampler.get_test() ] else: sample_weights = None resampled = [ self.resampler.get_train(), self.resampler.get_valid(), self.resampler.get_test() ] if self.member_number > 0: self.params.n_epochs = self.n_epochs_after_first if 'lr_after_first' in self.params.__dict__: self.params.optimizer['config'][ 'lr'] = self.params.lr_after_first if not self.use_sample_weights: sample_weights = None m = mlp.sequential_model( resampled, self.params, member_number=self.member_number, model_weights=self.weights, #the copy is because there is a bug in Keras that deletes names model_config=copy.deepcopy(self.model_config), sample_weight=sample_weights) self.weights = [l.get_weights() for l in m.layers] injection_index = self.incremental_index + self.member_number * len( self.incremental_layers) if self.incremental_layers is not None: if injection_index == -1: injection_index = len(self.model_config) new_layers = [] for i, l in enumerate(self.incremental_layers): l['config']['name'] = "DIB-incremental-{0}-{1}".format( self.member_number, i) new_layers.append(l) new_model_config = self.model_config[: injection_index] + new_layers + self.model_config[ injection_index:] self.model_config = copy.deepcopy(new_model_config) self.weights = self.weights[:injection_index] orig_train = self.resampler.get_train() K = orig_train[1].shape[1] errors = common.errors(m, orig_train[0], orig_train[1]) e = sum((errors * self.D)) / sum(errors + np.finfo(np.float32).eps) alpha = math.log((1 - e) / e + np.finfo(np.float32).eps) + math.log(K - 1) w = np.where(errors == 1, self.D * math.exp(alpha), self.D * math.exp(-alpha)) self.D = w / w.sum() self.resampler.update_weights(self.D) self.alphas.append(alpha) self.member_number += 1 m_yaml = m.to_yaml() m_weights = m.get_weights() del m return (m_yaml, m_weights)
for i in range(0,params.ensemble_size): print 'training member {0}'.format(i) members.append(method.create_member()) ensemble = method.create_aggregator(params,members,train_set,valid_set) test_set_x, test_set_y = method.resampler.get_test() test_score = accuracy(ensemble,test_set_x,test_set_y) print 'Intermediate test accuracy: {0} %'.format(test_score * 100.) intermediate_scores.append(test_score) final_score = test_score print "\nFinal Ensemble test accuracy: {0} %".format(final_score * 100.) print "Preparing distillation dataset.." train_set_yhat = ensemble.predict(dataset[0][0]) distilled_dataset = copy.deepcopy(dataset) distilled_dataset = ((dataset[0][0], train_set_yhat), dataset[1], dataset[2]) params = original_params mlp = sequential_model(distilled_dataset, params) exit(1) if 'results_db' in params.__dict__: if 'results_host' in params.__dict__: host = params.results_host else: host = None print "saving results to {0}@{1}".format(params.results_db,host) conn = MongoClient(host=host) db = conn[params.results_db] if 'results_table' in params.__dict__: table_name = params.results_table else: table_name = 'results' table = db[table_name] results = {
if m[0] is not None: members.append(m[:2]) ensemble = method.create_aggregator(params,members,train_set,valid_set) test_set_x, test_set_y = method.resampler.get_test() test_score = accuracy(ensemble,test_set_x,test_set_y) print('Intermediate test accuracy: {0} %'.format(test_score * 100.)) intermediate_scores.append(test_score) final_score = test_score if len(m) > 2 and not m[2]: #the ensemble method told us to stop break print("\nFinal Ensemble test accuracy: {0} %".format(final_score * 100.)) print("Preparing distillation dataset..") train_set_yhat = ensemble.predict(dataset[0][0]) distilled_dataset = ((dataset[0][0], train_set_yhat), dataset[1], dataset[2]) params = original_params mlp = sequential_model(distilled_dataset, params, model_yaml = members[-1][0]) if args.dump_shapes_to is not None: for i in range(len(members)): with open("{0}member-{1}.model".format(args.dump_shapes_to, i),"w") as f: f.truncate() f.write(members[i][0]) if 'results_db' in params.__dict__: if 'results_host' in params.__dict__: host = params.results_host else: host = None print("saving results to {0}@{1}".format(params.results_db,host)) conn = MongoClient(host=host) db = conn[params.results_db] if 'results_table' in params.__dict__: table_name = params.results_table
arg_param_pairings = [ (args.results_db, 'results_db'), (args.results_host, 'results_host'), (args.results_table, 'results_table'), (args.epochs, 'n_epochs'), ] if 'seed' in args.__dict__: print "setting random seed to: {0}".format(args.seed) numpy.random.seed(args.seed) from toupee import data from toupee import config from toupee.mlp import sequential_model import toupee print "using toupee version {0}".format(toupee.version) params = config.load_parameters(args.params_file) def arg_params(arg_value,param): if arg_value is not None: params.__dict__[param] = arg_value for arg, param in arg_param_pairings: arg_params(arg,param) dataset = data.load_data(params.dataset, pickled = params.pickled, one_hot_y = params.one_hot, join_train_and_valid = params.join_train_and_valid) mlp = sequential_model(dataset, params)
(args.results_host, 'results_host'), (args.results_table, 'results_table'), (args.epochs, 'n_epochs'), ] if 'seed' in args.__dict__: print(("setting random seed to: {0}".format(args.seed))) numpy.random.seed(args.seed) from toupee import data from toupee import config from toupee.mlp import sequential_model import toupee print(("using toupee version {0}".format(toupee.version))) params = config.load_parameters(args.params_file) def arg_params(arg_value,param): if arg_value is not None: params.__dict__[param] = arg_value for arg, param in arg_param_pairings: arg_params(arg,param) dataset = data.load_data(params.dataset, pickled = params.pickled, one_hot_y = params.one_hot, join_train_and_valid = params.join_train_and_valid, zca_whitening = params.zca_whitening) mlp = sequential_model(dataset, params) if args.save_file is not None: mlp.save(args.save_file)