def get_data_structure_info(result_struct): all_settings_ = result_struct.keys() first_result = result_struct.values()[0] all_measurements_ = remove_duplicates([ k for v in first_result.values() for k in v.keys() ]) # ['flops', 'error', ...] all_datasets_, all_subsets_, all_nets_ = [ remove_duplicates(all_category_entries) for all_category_entries in zip( *[categories for categories in first_result.keys()]) ] return all_settings_, all_measurements_, all_datasets_, all_subsets_, all_nets_
def test_remove_duplicates(): assert remove_duplicates(['a', 'b', 'a', 'c', 'c']) == ['a', 'b', 'c'] assert remove_duplicates(['a', 'b', 'a', 'c', 'c'], keep_last=True) == ['b', 'a', 'c'] assert remove_duplicates( ['Alfred', 'Bob', 'Cindy', 'Alina', 'Karol', 'Betty'], key=lambda x: x[0]) == ['Alfred', 'Bob', 'Cindy', 'Karol'] assert remove_duplicates( ['Alfred', 'Bob', 'Cindy', 'Alina', 'Karol', 'Betty'], key=lambda x: x[0], keep_last=True) == ['Cindy', 'Alina', 'Karol', 'Betty'] assert remove_duplicates( ['Alfred', 'Bob', 'Cindy', 'Alina', 'Karol', 'Betty'], key=lambda x: x[0], keep_last=True, hashable=False) == ['Cindy', 'Alina', 'Karol', 'Betty']
def get_best_score(score_info_pairs, subset = 'test', prediction_function = None, score_measure = None, lower_is_better = False): """ DEPRECATED!!! Given a list of (info, score) pairs which represet the progress over training, find the best score and return it. :param score_info_pairs: A list<(IterationInfo, dict)> of the type returned in train_and_test_online_predictor :param subset: 'train' or 'test' ... which subset to use to look for the best score :param prediction_function: Which prediction function (if there are multiple prediction functions, otherwise leave blank) :param score_measure: Which score measure (if there are multiple score measures, otherwise leave blank) :param lower_is_better: True if a lower score is better for the chosen score_measure :return: best_info, best_score best_info is an InterationInfo object best_score is a dict<(subset, prediction_function, score_measure) -> score> """ assert len(score_info_pairs)>0, "You need to have at least one score to determine the best one." _, first_score = score_info_pairs[0] all_subsets, all_functions, all_measures = [remove_duplicates(s) for s in zip(*first_score.keys())] if prediction_function is None: assert len(all_functions)==1, "You did not specify prediction_function... options are: {}".format(all_functions) prediction_function = all_functions[0] if score_measure is None: assert len(all_measures)==1, "You did not specify a score_measure... options are: {}".format(all_measures) score_measure = all_measures[0] best_info = None best_score = None for info, score in score_info_pairs: this_is_the_best = best_score is None or \ (score[subset, prediction_function, score_measure]<best_score[subset, prediction_function, score_measure]) == lower_is_better if this_is_the_best: best_score = score best_info = info return best_info, best_score
def get_record_invalid_arg_string(record, recursive=True, ignore_valid_keys=(), note_version = 'full'): """ Return a string identifying ig the arguments for this experiment are still valid. :return: """ assert note_version in ('full', 'short') experiment_id = record.get_experiment_id() if is_experiment_loadable(experiment_id): if record.info.has_field(ExpInfoFields.ARGS): last_run_args = OrderedDict([(k,v) for k,v in record.get_args().items() if k not in ignore_valid_keys]) current_args = OrderedDict([(k,v) for k,v in record.get_experiment().get_args().items() if k not in ignore_valid_keys]) if recursive: last_run_args = OrderedDict(flatten_struct(last_run_args, first_dict_is_namespace=True, primatives = PRIMATIVE_TYPES + (UnPicklableArg, ))) last_run_args = OrderedDict([(k, v) for k, v in last_run_args.items() if k not in ignore_valid_keys]) current_args = OrderedDict(flatten_struct(current_args, first_dict_is_namespace=True)) current_args = OrderedDict([(k, v) for k, v in current_args.items() if k not in ignore_valid_keys]) validity = record.args_valid(last_run_args=last_run_args, current_args=current_args) if validity is False: common, (old_args, new_args) = separate_common_items([list(last_run_args.items()), list(current_args.items())]) if len(old_args)+len(new_args)==0: raise Exception('Error displaying different args. Bug Peter.') all_changed_arg_names = remove_duplicates(list(name for name, _ in old_args)+list(name for name, _ in new_args)) changestr = ', '.join("{}:{}->{}".format(k, last_run_args[k] if k in last_run_args else '<N/A>', current_args[k] if k in current_args else '<N/A>') for k in all_changed_arg_names) notes = ("Change: " if note_version=='full' else "") + changestr elif validity is None: notes = "Cannot Determine: Unhashable Args" if note_version=='full' else '<Unhashable Args>' else: notes = "<No Change>" else: notes = "Cannot Determine: Inconsistent Experiment Record" if note_version == 'full' else '<Inconsistent Record>' else: notes = "Cannot Determine: Experiment Not Imported" if note_version=='full' else '<Not Imported>' return notes
def print_score_results(score, info=None): """ :param results: An OrderedDict in the format returned by assess_prediction_functions_on_generator. :return: """ if info is not None: print('Epoch {} (after {:.3g}s)'.format(info.epoch, info.time)) test_pair_names, function_names, cost_names = [remove_duplicates(k) for k in zip(*score.keys())] rows = build_table( lookup_fcn=lambda test_pair_name_and_function_name_, cost_name_: score[test_pair_name_and_function_name_[0], test_pair_name_and_function_name_[1], cost_name_], row_categories = [[test_pair_name for test_pair_name in test_pair_names], [function_name for function_name in function_names]], column_categories = [cost_name for cost_name in cost_names], row_header_labels=['Subset', 'Function'], clear_repeated_headers = False, remove_unchanging_cols=True ) import tabulate print(tabulate.tabulate(rows))
def get_costs(self): return remove_duplicates([c for _, c, c in self.scores.keys()])
def get_prediction_functions(self): return remove_duplicates([f for _, f, _ in self.scores.keys()])
def get_data_subsets(self): return remove_duplicates([s for s, _, _ in self.scores.keys()])
def get_record_invalid_arg_string(record, recursive=True, ignore_valid_keys=(), note_version='full'): """ Return a string identifying ig the arguments for this experiment are still valid. :return: """ assert note_version in ('full', 'short') experiment_id = record.get_experiment_id() if is_experiment_loadable(experiment_id): if record.info.has_field(ExpInfoFields.ARGS): last_run_args = OrderedDict([(k, v) for k, v in record.get_args().items() if k not in ignore_valid_keys]) current_args = OrderedDict([ (k, v) for k, v in record.get_experiment().get_args().items() if k not in ignore_valid_keys ]) if recursive: last_run_args = OrderedDict( flatten_struct(last_run_args, first_dict_is_namespace=True, primatives=PRIMATIVE_TYPES + (UnPicklableArg, ))) last_run_args = OrderedDict([(k, v) for k, v in last_run_args.items() if k not in ignore_valid_keys]) current_args = OrderedDict( flatten_struct(current_args, first_dict_is_namespace=True)) current_args = OrderedDict([(k, v) for k, v in current_args.items() if k not in ignore_valid_keys]) validity = record.args_valid(last_run_args=last_run_args, current_args=current_args) if validity is False: common, (old_args, new_args) = separate_common_items( [list(last_run_args.items()), list(current_args.items())]) if len(old_args) + len(new_args) == 0: raise Exception( 'Error displaying different args. Bug Peter.') all_changed_arg_names = remove_duplicates( list(name for name, _ in old_args) + list(name for name, _ in new_args)) changestr = ', '.join("{}:{}->{}".format( k, last_run_args[k] if k in last_run_args else '<N/A>', current_args[k] if k in current_args else '<N/A>') for k in all_changed_arg_names) notes = ("Change: " if note_version == 'full' else "") + changestr elif validity is None: notes = "Cannot Determine: Unhashable Args" if note_version == 'full' else '<Unhashable Args>' else: notes = "<No Change>" else: notes = "Cannot Determine: Inconsistent Experiment Record" if note_version == 'full' else '<Inconsistent Record>' else: notes = "Cannot Determine: Experiment Not Imported" if note_version == 'full' else '<Not Imported>' return notes
def test_remove_duplicates(): assert remove_duplicates(['a', 'b', 'a', 'c', 'c'])==['a', 'b', 'c'] assert remove_duplicates(['a', 'b', 'a', 'c', 'c'], keep_last=True)==['b', 'a', 'c'] assert remove_duplicates(['Alfred', 'Bob', 'Cindy', 'Alina', 'Karol', 'Betty'], key=lambda x: x[0])==['Alfred', 'Bob', 'Cindy', 'Karol'] assert remove_duplicates(['Alfred', 'Bob', 'Cindy', 'Alina', 'Karol', 'Betty'], key=lambda x: x[0], keep_last=True)==['Cindy', 'Alina', 'Karol', 'Betty'] assert remove_duplicates(['Alfred', 'Bob', 'Cindy', 'Alina', 'Karol', 'Betty'], key=lambda x: x[0], keep_last=True, hashable=False)==['Cindy', 'Alina', 'Karol', 'Betty']