def _adapt(self, info): # First, have an adaptive algorithm if self.n_initial_parameters == "grid": start = len(ParameterGrid(self.parameters)) else: start = self.n_initial_parameters def inverse(time): """ Decrease target number of models inversely with time """ return int(start / (1 + time)**self.decay_rate) example = toolz.first(info.values()) time_step = example[-1]["partial_fit_calls"] current_time_step = time_step + 1 next_time_step = current_time_step if inverse(current_time_step) == 0: # we'll never get out of here next_time_step = 1 while inverse(current_time_step) == inverse(next_time_step) and ( self.decay_rate and not self.patience or next_time_step - current_time_step < self.fits_per_score): next_time_step += 1 target = max(1, inverse(next_time_step)) best = toolz.topk(target, info, key=lambda k: info[k][-1]["score"]) if len(best) == 1: [best] = best return {best: 0} steps = next_time_step - current_time_step instructions = {b: steps for b in best} return instructions
def _adapt(self, info, first_step_completed=False): if all(v[-1]["partial_fit_calls"] == 1 for v in info.values()): # Do all the models have one partial fit call? self._steps = 0 if first_step_completed: # Sometimes, IncrementalSearchCV completes one step for us. We # recurse in this case -- see below for a note on the condition self._steps = 1 n, eta = self.n_initial_parameters, self.aggressiveness r = self.n_initial_iter n_i = int(math.floor(n * eta**-self._steps)) r_i = np.round(r * eta**self._steps).astype(int) if r_i == 1: # if r_i == 1, a step has already been completed for us (because # IncrementalSearchCV completes 1 partial_fit call automatically) return self._adapt(info, first_step_completed=True) best = toolz.topk(n_i, info, key=lambda k: info[k][-1]["score"]) self._steps += 1 if len(best) == 0: return {id_: 0 for id_ in info} pf_calls = {k: info[k][-1]["partial_fit_calls"] for k in best} additional_calls = {k: r_i - pf_calls[k] for k in best} return additional_calls
def tune_tolerance(self): """ Tune tolerance (1 - threshold) for setting as the default API parameter (Uses GPU to reduce computation from ~ 1 hour to a matter of seconds) """ if len(self.val_generator) > 1: warnings.warn( 'Less than the entire validation batch will be used...') images, truth_masks = self.val_generator.__getitem__(0) print( f'Tuning of the tolerance parameter will occur on {images.shape[0]} images.' ) predictions = self.model.predict(images) tolerances = np.linspace(0.02, 1, 50) results = self._calculate_f1_scores(tolerances, truth_masks, predictions) self._plot_tolerances(tolerances, results) populations = [(f'{tolerance:.2f}', np.median(scores), np.std(scores)) for tolerance, scores in zip(tolerances, results)] tolerance, f1_median, f1_stdev = min(topk(5, populations, key=second), key=third) print( f'Tuned tolerance: {tolerance} w/ median={f1_median:.4f} stdev={f1_stdev:.4f}' ) return float(tolerance)
def _additional_calls(self, info): # First, have an adaptive algorithm if self.n_initial_parameters == "grid": start = len(ParameterGrid(self.parameters)) else: start = self.n_initial_parameters def inverse(time): """ Decrease target number of models inversely with time """ return int(start / (1 + time)**self.decay_rate) example = toolz.first(info.values()) time_step = example[-1]["partial_fit_calls"] current_time_step = time_step + 1 next_time_step = current_time_step if inverse(current_time_step) == 0: # we'll never get out of here next_time_step = 1 while inverse(current_time_step) == inverse(next_time_step) and ( self.decay_rate and not self.patience or next_time_step - current_time_step < self.scores_per_fit): next_time_step += 1 target = max(1, inverse(next_time_step)) best = toolz.topk(target, info, key=lambda k: info[k][-1]["score"]) if len(best) == 1: [best] = best return {best: 0} steps = next_time_step - current_time_step instructions = {b: steps for b in best} # Second, stop on plateau if any models have already converged out = {} for k, steps in instructions.items(): records = info[k] current_calls = records[-1]["partial_fit_calls"] if self.max_iter and current_calls >= self.max_iter: out[k] = 0 elif self.patience and current_calls >= self.patience: plateau = [ h["score"] for h in records if current_calls - h["partial_fit_calls"] <= self.patience ] if all(score <= plateau[0] + self.tol for score in plateau[1:]): out[k] = 0 else: out[k] = steps else: out[k] = steps return out
def _additional_calls(self, info): if self.n_initial_parameters == "grid": start = len(ParameterGrid(self.parameters)) else: start = self.n_initial_parameters def inverse(time): """ Decrease target number of models inversely with time """ return int(start / (1 + time)**self.decay_rate) example = toolz.first(info.values()) time_step = example[-1]["partial_fit_calls"] current_time_step = time_step + 1 next_time_step = current_time_step if inverse(current_time_step) == 0: # we'll never get out of here next_time_step = 1 while inverse(current_time_step) == inverse(next_time_step) and ( not self.patience or next_time_step - current_time_step < self.scores_per_fit): next_time_step += 1 target = max(1, inverse(next_time_step)) best = toolz.topk(target, info, key=lambda k: info[k][-1]["score"]) if len(best) == 1: [best] = best return {best: 0} out = {} for k in best: records = info[k] if self.max_iter and len(records) >= self.max_iter: out[k] = 0 elif self.patience and len(records) >= self.patience: old = records[-self.patience]["score"] if all(d["score"] < old + self.tol for d in records[-self.patience:]): out[k] = 0 else: out[k] = next_time_step - current_time_step else: out[k] = next_time_step - current_time_step return out
def keys_to_flush(lengths, fraction=0.1, maxcount=100000): """ Which keys to remove >>> lengths = {'a': 20, 'b': 10, 'c': 15, 'd': 15, ... 'e': 10, 'f': 25, 'g': 5} >>> keys_to_flush(lengths, 0.5) ['f', 'a'] """ top = topk(max(len(lengths) // 2, 1), lengths.items(), key=1) total = sum(lengths.values()) cutoff = min( maxcount, max(1, bisect(list(accumulate(add, pluck(1, top))), total * fraction))) result = [k for k, v in top[:cutoff]] assert result return result
def keys_to_flush(lengths, fraction=0.1, maxcount=100000): """ Which keys to remove >>> lengths = {'a': 20, 'b': 10, 'c': 15, 'd': 15, ... 'e': 10, 'f': 25, 'g': 5} >>> keys_to_flush(lengths, 0.5) ['f', 'a'] """ top = topk(max(len(lengths) // 2, 1), lengths.items(), key=1) total = sum(lengths.values()) cutoff = min(maxcount, max(1, bisect(list(accumulate(add, pluck(1, top))), total * fraction))) result = [k for k, v in top[:cutoff]] assert result return result
def get_closest_indices(new_sent, feats, k=3): """ Parameters ---------- new_sent : str New sentence feats : array-like, 2d Features for different sentences. Returns ------- idx : int Index of ``feats`` that is closest to ``new_sent``. """ model = initialize(download=False) feat = model.encode([new_sent], bsize=128, tokenize=False) dists = cdist(feat, feats, "cosine") assert dists.shape[0] == 1 dists = dists[0] vals = topk(k, -dists) idxs = [k for k, v in enumerate(dists) if -v in vals] return idxs[::-1]
def topk_dict(d, k=10): return dict(toolz.topk(k, d.items(), key=lambda x: x[1]))
def balance(self): with log_errors(): i = 0 s = self.scheduler occupancy = s.occupancy idle = s.idle saturated = s.saturated if not idle or len(idle) == len(self.scheduler.workers): return log = list() start = time() broken = False seen = False acted = False if not s.saturated: saturated = topk(10, s.workers, key=occupancy.get) saturated = [ w for w in saturated if occupancy[w] > 0.2 and len(s.processing[w]) > s.ncores[w] ] elif len(s.saturated) < 20: saturated = sorted(saturated, key=occupancy.get, reverse=True) if len(idle) < 20: idle = sorted(idle, key=occupancy.get) for level, cost_multiplier in enumerate(self.cost_multipliers): if not idle: break for sat in list(saturated): stealable = self.stealable[sat][level] if not stealable or not idle: continue else: seen = True for key in list(stealable): i += 1 if not idle: break idl = idle[i % len(idle)] duration = s.processing[sat][key] if (occupancy[idl] + cost_multiplier * duration <= occupancy[sat] - duration / 2): self.move_task(key, sat, idl) log.append((start, level, key, duration, sat, occupancy[sat], idl, occupancy[idl])) self.scheduler.check_idle_saturated(sat) self.scheduler.check_idle_saturated(idl) seen = True if self.cost_multipliers[ level] < 20: # don't steal from public at cost stealable = self.stealable_all[level] if stealable: seen = True for key in list(stealable): if not idle: break sat = s.rprocessing[key] if occupancy[sat] < 0.2: continue if len(s.processing[sat]) <= s.ncores[sat]: continue i += 1 idl = idle[i % len(idle)] duration = s.processing[sat][key] if (occupancy[idl] + cost_multiplier * duration <= occupancy[sat] - duration / 2): self.move_task(key, sat, idl) log.append((start, level, key, duration, sat, occupancy[sat], idl, occupancy[idl])) self.scheduler.check_idle_saturated(sat) self.scheduler.check_idle_saturated(idl) seen = True if seen and not acted: break if log: self.log.append(log) self.count += 1 stop = time() if self.scheduler.digests: self.scheduler.digests['steal-duration'].add(stop - start)
def run(self): print("Evolution has been launched") start_run_time = self.current_secs_time() population = Population([ CNNGenome([ AugmentationGene(), SequentialModelGene(), OptimizerGene(), OutputActivationGene() ]), CNNGenome([ AugmentationGene(), SequentialModelGene(), OptimizerGene(), OutputActivationGene() ]), CNNGenome([ AugmentationGene(), SequentialModelGene(), OptimizerGene(), OutputActivationGene() ]), CNNGenome([ AugmentationGene(), SequentialModelGene(), OptimizerGene(), OutputActivationGene() ]), CNNGenome([ AugmentationGene(), SequentialModelGene(), OptimizerGene(), OutputActivationGene() ]) ]) # Get timebox segments reversed ( descending lengths) timeboxes = self.calculate_exp_segments(self.__number_of_evolutions, self.__max_runtime)[::-1] # Get databox segments reversed ( descending lengths) databoxes = list( map( round, self.calculate_exp_segments( self.__number_of_evolutions, self.__data_context.train_nrows()))) assert sum(databoxes) == self.__data_context.train_nrows() best_individual = None evaluator = Evaluator() for (evolution_number, timebox), databox in zip(enumerate(timeboxes, 1), databoxes): generation_number = 1 start_evolution_time = self.current_secs_time() X_train_smpl, Y_train_smpl = self.databox_train_sample(databox) while self.current_secs_time() - start_evolution_time < timebox \ and self.current_secs_time() - start_run_time < self.__max_runtime: # Split randomly into train and the validation set for the fitting sample_data_ctx = self.get_shuffled_sample_data_ctx( X_train_smpl, Y_train_smpl) #Build population individuals = list( map(lambda cnn_ind: cnn_ind.build(self.__seed), population.get_individuals())) print("Individuals" + str(individuals)) # Evaluate population ev_individuals = list( map( lambda cnn_ind: evaluator.evaluate( cnn_ind, sample_data_ctx), individuals)) # Print evaluation results self.__print_ev_individuals(ev_individuals) # TODO Select parents for cross-over and mutations. Note minus sign to pick smallest values parents = toolz.topk(3, ev_individuals, key=lambda ev_individual: -ev_individual. get_fitness().get_valid_loss()) self.__print_ev_individuals(parents) # Crossover or mutate individuals mutator = Mutator() offspring_genomes = list( map( lambda parent: mutator.mutate(parent. get_original_genome()), copy.deepcopy(parents))) # Materialize offspring offspring = list( map( lambda offspring_genome: offspring_genome.build( self.__seed), offspring_genomes)) # Evaluate offspring ev_offspring = list( map( lambda offspring_ind: evaluator.evaluate( offspring_ind, sample_data_ctx), offspring)) print("\n Evaluated offspring \n") self.__print_ev_individuals(ev_offspring) # Combine original population and offspring expanded_individuals = ev_individuals + ev_offspring # Run survival phase for evaluated individuals survived = toolz.topk(len(individuals), expanded_individuals, key=lambda ev_individual: -ev_individual. get_fitness().get_valid_loss()) # Update best individual best_individual = toolz.topk( 1, survived, key=lambda ev_individual: -ev_individual.get_fitness( ).get_valid_loss()) survived_genomes = list( map( lambda survived_ind: survived_ind.get_original_genome( ), survived)) population = Population(individuals=survived_genomes) print("Evolution #{} | generation #{} is finished".format( evolution_number, generation_number)) generation_number += 1 # Evaluate best individual on whole data total_sample_data_ctx = self.get_shuffled_sample_data_ctx( *self.__data_context.get_train()) best_individual_evaluated = evaluator.evaluate( best_individual[0].get_individual(), total_sample_data_ctx) return (population, best_individual_evaluated)