def score_over_layers(models, random, labels, bench, convergence=True, ax=None): if bench is not None: benchmarks = bench conn = get_connection() names = [] if convergence and 'CORnet-S_full' in convergence_epoch: full_tr = load_scores(conn, [f'CORnet-S_full_epoch_{convergence_epoch["CORnet-S_full"]:02d}'], benchmarks)[ f'CORnet-S_full_epoch_{convergence_epoch["CORnet-S_full"]:02d}'] else: full_tr = load_scores(conn, ['CORnet-S_full_epoch_06'], benchmarks)['CORnet-S_full_epoch_06'] model_dict = load_error_bared(conn, list(chain(models.keys(), random.keys())), benchmarks, convergence=convergence) if len(benchmarks) < 6: benchmarks_labels = ['V4', 'IT', 'Behavior', 'Imagenet'] else: benchmarks_labels = ['V1', 'V2', 'V4', 'IT', 'Behavior', 'Imagenet'] data = {} err = {} x_ticks = {} for i in range(len(benchmarks)): data[benchmarks_labels[i]] = [] err[benchmarks_labels[i]] = [] x_ticks[benchmarks_labels[i]] = [] layers = [] for model, layer in models.items(): layers.append(layer_best_2[layer]) frac = (model_dict[model][i] / full_tr[i]) * 100 frac_err = (model_dict[model][len(benchmarks):][i] / full_tr[i]) * 100 data[benchmarks_labels[i]].append(frac) err[benchmarks_labels[i]].append(frac_err) x_ticks[benchmarks_labels[i]] = layers plot_data_double(data, data2=None, err=err, name=f'Artificial Genome + Critical Training', x_name='Number of trained layers', y_name=r'Benchmark Score [% of standard training]', x_ticks=x_ticks, x_ticks_2=[], percent=True, ax=ax, pal=red_palette, annotate_pos=1)
def image_scores(models, imgs, labels, ax=None, selection=[]): names = [] conn = get_connection() for model in models: for img in imgs: name = f'{model}_img{img}' names.append(f'{name}_epoch_{convergence_images[name]}') if model == 'CORnet-S_cluster2_v2_IT_trconv3_bi': model = f'{model}_seed42' names.append(f'{model}_epoch_{convergence_epoch[model]}') names.append('CORnet-S_full_epoch_43') model_dict = load_scores(conn, names, benchmarks) data2 = {} full = np.mean(model_dict['CORnet-S_full_epoch_43'][selection]) for i in imgs: for model, name in zip(models, labels): data2[name] = [] name1 = f'{model}_img{i}' frac = (np.mean(model_dict[f'{name1}_epoch_{convergence_images[name1]}'][selection]) / full) * 100 data2[name].append(frac) if model == 'CORnet-S_cluster2_v2_IT_trconv3_bi': model = f'{model}_seed42' frac = (np.mean(model_dict[f'{model}_epoch_{convergence_epoch[model]}'][selection]) / full) * 100 data2[name].append(frac) if len(selection) == 1: title = 'Imagenet score vs number of weights' y = r'Imagenet [% of standard training]' else: title = f'Brain scores mean vs number of weights' y = r'Brain Predictivity [% of standard training]' imgs.append(1200000) plot_data_double(data2, {}, '', x_name='Number of images in million', y_name=y, x_ticks={'IT init, selective training': imgs}, x_ticks_2=imgs, percent=True, log=True, ax=ax, million=True)
def plot_first_epochs(models, epochs=None, brain=True, convergence=True, ax=None): model_dict = {} conn = get_connection() if epochs is None: epochs = (0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1, 2, 3, 4, 5, 6) data = {} x_values = {} if convergence and 'CORnet-S_full' in convergence_epoch: full_tr = load_scores(conn, [f'CORnet-S_full_epoch_{convergence_epoch["CORnet-S_full"]:02d}'], benchmarks)[ f'CORnet-S_full_epoch_{convergence_epoch["CORnet-S_full"]:02d}'] else: full_tr = load_scores(conn, ['CORnet-S_full_epoch_06'], benchmarks)['CORnet-S_full_epoch_06'] for model, name in models.items(): names = [] for epoch in epochs: if epoch % 1 == 0: names.append(f'{model}_epoch_{epoch:02d}') else: names.append(f'{model}_epoch_{epoch:.1f}') if convergence and model in convergence_epoch: names.append(f'{model}_epoch_{convergence_epoch[model]:02d}') model_dict = load_scores(conn, names, benchmarks) scores = [] for epoch in epochs: if brain: full = np.mean(full_tr[2:5]) if epoch % 1 == 0: frac = (np.mean(model_dict[f'{model}_epoch_{int(epoch):02d}'][2:5]) / full) * 100 scores.append(frac) else: frac = (np.mean(model_dict[f'{model}_epoch_{epoch:.1f}'][2:5]) / full) * 100 scores.append(frac) else: full = np.mean(full_tr[5]) if epoch % 1 == 0: frac = (np.mean(model_dict[f'{model}_epoch_{int(epoch):02d}'][5]) / full) * 100 scores.append(frac) else: frac = (np.mean(model_dict[f'{model}_epoch_{epoch:.1f}'][5]) / full) * 100 scores.append(frac) if convergence and model in convergence_epoch: if brain: frac = (np.mean(model_dict[f'{model}_epoch_{convergence_epoch[model]:02d}'][2:5]) / full) * 100 scores.append(frac) else: frac = (np.mean(model_dict[f'{model}_epoch_{convergence_epoch[model]:02d}'][5]) / full) * 100 scores.append(frac) x_values[name] = epochs + [convergence_epoch[model]] else: x_values[name] = epochs data[name] = scores title = f'Brain scores mean vs epochs' if brain else 'Imagenet score vs epochs' plot_data_base(data, 'First epochs', x_values, 'Epochs', 'Brain Predictivity [% of standard training]', x_ticks=epochs + [30, 40, 50], log=True, percent=True, special_xaxis=True, legend=False, only_blue=False, palette=red_palette, annotate=True, annotate_pos=1, ax=ax)
def score_layer_depth(values, brain=True): names = [] conn = get_connection() for k, v in values.items(): names.append(f'{k}_epoch_05') for k, v in random_scores.items(): if k != 'CORnet-S_random' and k != 'CORnet-S_train_random': names.append(f'{k}_epoch_05') else: names.append(k) model_dict = load_scores(conn, names, ['movshon.FreemanZiemba2013.V1-pls', 'movshon.FreemanZiemba2013.V2-pls', 'dicarlo.Majaj2015.V4-pls', 'dicarlo.Majaj2015.IT-pls', 'dicarlo.Rajalingham2018-i2n', 'fei-fei.Deng2009-top1']) weight_num = [9408, 36864, 8192, 16384, 65536, 2359296, 65536, 32768, 65536, 262144, 9437184, 262144, 131072, 262144, 1048576, 37748736, 1048576, 512000] acc = [52860096 + 512000] for i in weight_num: acc.append(acc[-1] - i) weights = [] results = [] for model, l in values.items(): index = layers.index(l) weights.append(acc[index]) res = model_dict[f'{model}_epoch_05'] if brain: results.append(np.mean(res[2:4])) # if index < 7: # results.append(np.mean(res[0:1])) # else: # results.append(np.mean(res[0:2])) else: results.append(res[5]) rand_names = [] for model, l in random_scores.items(): index = layers.index(l) weights.append(acc[index]) if model != 'CORnet-S_random' and model != 'CORnet-S_train_random': res = model_dict[f'{model}_epoch_05'] else: res = model_dict[model] if brain: results.append(np.mean(res[0:4])) # if index < 7: # results.append(np.mean(res[0:1])) # else: # results.append(np.mean(res[0:2])) else: results.append(res[5]) rand_names.append(f'Random {l}') title = f'Brain scores mean vs number of weights' if brain else 'Imagenet score vs number of weights' scatter_plot(weights, results, x_label='Num of weights', y_label='Score', labels=list(values.values()) + rand_names, title=title)
def plot_first_epochs(models, epochs=None, brain=True, convergence=True, model_name='resnet'): model_dict = {} conn = get_connection() if epochs is None: epochs = (0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1, 2, 3, 4, 5, 6) data = {} x_values = {} conv_number = convergence_epoch[f'{model_name}_v1_CORnet-S_full'] full_tr = load_scores(conn, [f'{model_name}_v1_CORnet-S_full_epoch_{conv_number}'], benchmarks)[ f'{model_name}_v1_CORnet-S_full_epoch_{conv_number}'] for model, name in models.items(): names = [] for epoch in epochs: if epoch % 1 == 0: names.append(f'{model}_epoch_{epoch:02d}') else: names.append(f'{model}_epoch_{epoch:.1f}') if convergence and model in convergence_epoch: names.append(f'{model}_epoch_{convergence_epoch[model]:02d}') model_dict = load_scores(conn, names, benchmarks) scores = [] for epoch in epochs: if brain: full = np.mean(full_tr[2:5]) if epoch % 1 == 0: frac = (np.mean(model_dict[f'{model}_epoch_{int(epoch):02d}'][2:5]) / full) * 100 scores.append(frac) else: frac = (np.mean(model_dict[f'{model}_epoch_{epoch:.1f}'][2:5]) / full) * 100 scores.append(frac) else: full = np.mean(full_tr[5]) if epoch % 1 == 0: frac = (np.mean(model_dict[f'{model}_epoch_{int(epoch):02d}'][5]) / full) * 100 scores.append(frac) else: frac = (np.mean(model_dict[f'{model}_epoch_{epoch:.1f}'][5]) / full) * 100 scores.append(frac) if convergence and model in convergence_epoch: if brain: frac = (np.mean(model_dict[f'{model}_epoch_{convergence_epoch[model]:02d}'][2:5]) / full) * 100 scores.append(frac) y = 'mean(V4, IT, Behavior) [% of standard training]' else: frac = (np.mean(model_dict[f'{model}_epoch_{convergence_epoch[model]:02d}'][5]) / full) * 100 y = 'Imagenet [% of standard training]' scores.append(frac) x_values[name] = epochs + [convergence_epoch[model]] else: x_values[name] = epochs data[name] = scores title = f'{model_name} Brain scores mean vs epochs' if brain else f'{model_name} Imagenet score vs epochs' plot_data_base(data, '', x_values, 'Epochs', y, x_ticks=epochs + [10, 20, 30], percent=True, special_xaxis=True, only_blue=False)
def plot_models_benchmarks(models, file_name, benchmarks, convergence=True, gs=None, ax=None): conn = get_connection() model_dict = load_error_bared(conn, models.keys(), benchmarks) if len(benchmarks) < 6: benchmarks_labels = ['V4', 'IT', 'Behavior', 'Imagenet'] else: benchmarks_labels = ['V1', 'V2', 'V4', 'IT', 'Behavior'] # 'Imagenet' data_set = {} err = {} # We replace the model id, a more human readable version for id, desc in models.items(): data_set[desc] = model_dict[id][:5] err[desc] = model_dict[id][6:-1] plot_bar_benchmarks(data_set, benchmarks_labels, '', r'\textbf{Scores}', file_name, yerr=err, gs=gs, ax=ax)
def score_over_layers_avg(models_resnet, random, models_alexnet={}, random_alexnet={}, imagenet=False, convergence=False, model_name='resnet', layers_numbers=[layers_number, layers_number, layers_number_mobilenet, layers_number_mobilenet], gs=None, ax=None, selection=[]): conn = get_connection() full = 0 model_dict = load_error_bared(conn, list( chain(models_resnet.keys(), models_mobilenet.keys(), random.keys(), random_mobilenet.keys())), benchmarks, convergence=convergence) data = {} err = {} layers = {} labels = {} idx = 0 for models, label in zip([random, models_resnet], # , random_mobilenet, models_mobilenet ['Resnet50 KN+DT', 'Resnet50 Transfer AG+CT', 'Alexnet KN+DT', 'Alexnet Transfer AG+CT']): data[label] = [] layers[label] = [] layers_number = layers_numbers[idx] idx += 1 for model, layer in models.items(): layers[label].append(layers_number[layer]) if model == f'{model_name}_v1_CORnet-S_full': full = np.mean(model_dict[model][selection]) data[label].append(100) else: percent = (np.mean(model_dict[model][selection]) / full) * 100 data[label].append(percent) full_err = (np.mean(model_dict[model][:6][selection]) / full) * 100 err[label] = full_err if 'Alexnet' in label: labels[label] = models.values() else: labels[label] = [value.split('.')[0] for value in models.values()] if imagenet: title = f'{model_name} Imagenet score over layers' y = 'Imagenet [% of standard training]' else: title = f'{model_name} Brain-Score Benchmark mean(V4, IT, Behavior) over layers' if len(selection) == 3: y = r"mean(V4, IT, Behavior) [% of standard training]" else: y = r"mean(V1,V2,V4,IT,Behavior) [% of standard training]" plot_data_double(data, {}, '', x_name='Number of trained layers [% of all layers]', y_name=y, x_ticks=layers, x_ticks_2=[], percent=False, percent_x=True, pal=['#424949'] + [green_palette[1]] + ['#ABB2B9'] + [green_palette[0]], data_labels=labels, gs=gs, ax=ax)
def plot_models_vs(models, file_name, convergence=False, epoch=0, imagenet=False, gs=None, ax=None, selection=[]): model_dict = {} conn = get_connection() names = [] for name, mod in models.items(): for model in mod.values(): # if convergence: # epoch = convergence_epoch[model] names.append(model) model_dict = load_error_bared(conn, names, benchmarks, convergence=convergence, epochs=[0]) labels = [] data_set = {} err = {} # We replace the model id, a more human readable version for name, models in models.items(): labels.append(name) # data_set[name] = {} for model_name, model in models.items(): if model_name not in data_set: data_set[model_name] = [] err[model_name] = [] # if imagenet: # data_set[model_name].append(np.mean(model_dict[f'{model}_epoch_{epoch:02d}'][5])) # else: if convergence: data_set[model_name].append(np.mean(model_dict[model][selection])) err[model_name].append(np.mean(model_dict[model][6:][selection])) else: data_set[model_name].append(np.mean(model_dict[f'{model}_epoch_{epoch:02d}'][selection])) err[model_name].append(np.mean(model_dict[f'{model}_epoch_{epoch:02d}'][6:][selection])) # if convergence and 'CORnet-S_full' in convergence_epoch: full_tr = load_error_bared(conn, [f'CORnet-S_full'], benchmarks)[ f'CORnet-S_full'] # else: # full_tr = load_scores(conn, ['CORnet-S_full_epoch_0'], benchmarks)['CORnet-S_full_epoch_06'] # print(f'Mean of brain benchmark model {desc}, {np.mean(data_set[desc][2:5])}') # if imagenet: # line =np.mean(full_tr[5]) # else: line = np.mean(full_tr[selection]) if len(selection) == 3: y = r"\textbf{Brain Predictivity} " else: y = r"\textbf{Brain Predictivity}" pals = blue_palette plot_bar_benchmarks(data_set, labels, '', y, file_name, yerr=err, line=line, label=True, grey=False, gs=gs, ax=ax)
def plot_model_avg_benchmarks(models, file_name): model_dict = {} conn = get_connection() epoch = 6 names = [] for model in models.keys(): names.append(f'{model}_epoch_{epoch:02d}') model_dict = load_scores(conn, names, benchmarks) benchmarks_labels = ['Brain Predictivity', 'Imagenet'] data_set = {} # We replace the model id, a more human readable version for id, desc in models.items(): data = model_dict[f'{id}_epoch_{epoch:02d}'] data_set[desc] = [np.mean(data[0:5]), data[5]] print(f'Mean of brain benchmark model {desc}, {np.mean(data[0:5])}') plot_bar_benchmarks(data_set, benchmarks_labels, '', 'Scores', file_name)
def plot_benchmarks_over_epochs(model, epochs=None, benchmarks=benchmarks, selection=[2, 3, 4], ax=None): # TODO: Add error bars model_dict = {} conn = get_connection() if epochs is None: epochs = (0, 5, 10, 15, 20) names = [] # for epoch in epochs: # if epoch % 1 == 0: # names.append(f'{model}_epoch_{epoch:02d}') # else: # names.append(f'{model}_epoch_{epoch:.1f}') model_dict = load_error_bared(conn, [model, 'CORnet-S_full'], benchmarks, epochs=epochs, convergence=True) full = model_dict['CORnet-S_full'] benchmarks_labels = ['V1', 'V2', 'V4', 'IT', 'Behavior', 'Mean'] data = {} for i in range(len(benchmarks) - 1): if i in selection: data[benchmarks_labels[i]] = [] for epoch in epochs: if epoch % 1 == 0: frac = (model_dict[f'{model}_epoch_{epoch:02d}'][i] / full[i]) * 100 else: frac = (model_dict[f'{model}_epoch_{epoch:.1f}'][i] / full[i]) * 100 data[benchmarks_labels[i]].append(frac) end = (np.mean(model_dict[model][i]) / np.mean(full[i])) * 100 data[benchmarks_labels[i]].append(end) data[benchmarks_labels[-1]] = [] for epoch in epochs: if epoch % 1 == 0: frac = (np.mean(model_dict[f'{model}_epoch_{epoch:02d}'][selection]) / np.mean(full[selection])) * 100 else: frac = (np.mean(model_dict[f'{model}_epoch_{epoch:.1f}'][selection]) / np.mean(full[selection])) * 100 data[benchmarks_labels[-1]].append(frac) end = (np.mean(model_dict[model][selection]) / np.mean(full[selection])) * 100 data[benchmarks_labels[-1]].append(end) plot_data_base(data, f'', r'\textbf{Epoch}', r'\textbf{Score} [\% of standard training]', epochs + [43], x_ticks=[value for value in epochs if value not in [0.1, 0.2, 0.3, 0.4, 0.6, 0.7, 0.8, 0.9, 15]] + [ 43], x_labels=[value for value in epochs if value not in [0.1, 0.2, 0.3, 0.4, 0.6, 0.7, 0.8, 0.9, 15]] + [ 'Conv'], percent=True, alpha=0.5, log=True, annotate=True, legend=False, annotate_pos=2, ax=ax, palette=grey_palette[:len(benchmarks_labels) - 1] + [blue_palette[0]])
def plot_models_benchmark_vs_public(models, file_name): model_dict = {} conn = get_connection() epoch = 6 names = [] for model in models.keys(): names.append(f'{model}_epoch_{epoch:02d}') model_dict = load_scores(conn, names, benchmarks) model_dict_pub = load_scores(conn, names, benchmarks_public) benchmarks_labels = ['V1', 'V2', 'V4', 'IT', 'Behavior', 'Imagenet'] data_set = {} # We replace the model id, a more human readable version for id, desc in models.items(): data_set[desc] = model_dict[f'{id}_epoch_{epoch:02d}'] data_set[f'{desc} public'] = model_dict_pub[f'{id}_epoch_{epoch:02d}'] print(f'Mean of brain benchmark model {desc}, {np.mean(data_set[desc][2:5])}') plot_bar_benchmarks(data_set, benchmarks_labels, 'Model scores in epoch 6', 'Score [% of standard training]', file_name, grey=True)
def plot_single_benchmarks(models, epochs=None, compare_batchfix=False, run_mean=False): model_dict = {} conn = get_connection() if epochs is None: epochs = (0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1, 2, 3, 4, 5, 6) data = {} benchmarks_label = ['V1', 'V2', 'V4', 'IT', 'Behavior', 'Imagenet'] for model, name in models.items(): names = [] for epoch in epochs: if epoch % 1 == 0: names.append(f'{model}_epoch_{epoch:02d}') else: names.append(f'{model}_epoch_{epoch:.1f}') if compare_batchfix: names.append(f'{model}_epoch_{epoch:02d}_BF') model_dict[name] = load_scores(conn, names, benchmarks) for i in range(6): for model, name in models.items(): scores = [] for epoch in epochs: if epoch % 1 == 0: scores.append(model_dict[name][f'{model}_epoch_{int(epoch):02d}'][i]) else: scores.append(model_dict[name][f'{model}_epoch_{epoch:.1f}'][i]) if run_mean: data[name] = [scores[0]] + np.convolve(scores, np.ones((3,)) / 3, mode='valid') + [scores[-1]] else: data[name] = scores if compare_batchfix: scores = [] for epoch in epochs: scores.append(model_dict[name][f'{model}_BF_epoch_{epoch:02d}'][i]) if run_mean: data[f'{name}_BF'] = np.convolve(scores, np.ones((3,)) / 3, mode='same') else: data[f'{name}_BF'] = scores title = f'{benchmarks_label[i]} benchmark vs epochs' plot_data_base(data, title, epochs, 'Epoch', 'Score', x_ticks=epochs, log=True)
def delta_heatmap(model1, model2, imgs, epochs, selection=[], title='', ax=None): names = [] conn = get_connection() for model in [model1, model2]: if model == 'CORnet-S_cluster2_v2_IT_trconv3_bi': model_spec = model else: model_spec = model for img in imgs: name = f'{model}_img{img}' for epoch in epochs: names.append(f'{name}_epoch_{epoch:02d}') names.append(f'{name}_epoch_{convergence_images[name]}') names.append(f'{model_spec}_epoch_{convergence_epoch[model_spec]}') for epoch in epochs: names.append(f'{model}_epoch_{epoch:02d}') names.append('CORnet-S_full_epoch_43') model_dict = load_scores(conn, names, benchmarks) full = np.mean(model_dict['CORnet-S_full_epoch_43'][selection]) matrix = np.zeros([len(imgs) + 1, len(epochs) + 1]) data = {} for i in range(len(imgs)): for j in range(len(epochs)): name1 = f'{model1}_img{imgs[i]}_epoch_{epochs[j]:02d}' name2 = f'{model2}_img{imgs[i]}_epoch_{epochs[j]:02d}' matrix[i, j] = calc_dif(name1, name2, model_dict, full, selection) name = f'{model1}_img{imgs[i]}' name = f'{name}_epoch_{convergence_images[name]:02d}' name2 = f'{model2}_img{imgs[i]}' name2 = f'{name2}_epoch_{convergence_images[name2]:02d}' matrix[i, -1] = calc_dif(name, name2, model_dict, full, selection) names.append(f'{model1}_epoch_{convergence_epoch[model1]:02d}') for j in range(len(epochs)): name1 = f'{model1}_epoch_{epochs[j]:02d}' name2 = f'{model2}_epoch_{epochs[j]:02d}' matrix[-1, j] = calc_dif(name1, name2, model_dict, full, selection) name = f'CORnet-S_cluster2_v2_IT_trconv3_bi_epoch_{convergence_epoch["CORnet-S_cluster2_v2_IT_trconv3_bi"]:02d}' name2 = f'{model2}_epoch_{convergence_epoch[model2]:02d}' matrix[-1, -1] = calc_dif(name, name2, model_dict, full, selection) plot_heatmap(matrix, r'\textbf{Epochs}', r'\textbf{Images}', title=title, annot=True, ax=ax, cbar=False, cmap='RdYlGn', percent=False, alpha=0.6, fmt='.0%', vmin=-0.30, vmax=0.30, yticklabels=imgs + ['All'], xticklabels=epochs + ['Convergence'])
def image_epoch_heatmap(model, imgs, epochs, selection=[], title=r'\textbf{Standard training epochs/images trade-off}', ax=None): names = [] conn = get_connection() # delta= 'CORnet-S_full' # if model == 'CORnet-S_cluster2_v2_IT_trconv3_bi': # model1 = model # else: # model1 = model for img in imgs: name = f'{model}_img{img}' # for epoch in epochs: # names.append(f'{name}_epoch_{epoch:02d}') names.append(name) names.append(model) # for epoch in epochs: # names.append(f'{model}_epoch_{epoch:02d}') names.append('CORnet-S_full') model_dict = load_error_bared(conn, names, epochs=epochs, benchmarks=benchmarks) full = np.mean(model_dict['CORnet-S_full'][selection]) matrix = np.zeros([len(imgs) + 1, len(epochs) + 1]) data = {} for i in range(len(imgs)): for j in range(len(epochs)): name1 = f'{model}_img{imgs[i]}_epoch_{epochs[j]:02d}' frac = (np.mean(model_dict[name1][selection]) / full) matrix[i, j] = frac name = f'{model}_img{imgs[i]}' # name = f'{name}_epoch_{convergence_images[name]:02d}' frac = (np.mean(model_dict[name][selection]) / full) matrix[i, -1] = frac # names.append(f'{model1}_epoch_{convergence_epoch[model1]:02d}') for j in range(len(epochs)): name1 = f'{model}_epoch_{epochs[j]:02d}' frac = (np.mean(model_dict[name1][selection]) / full) matrix[-1, j] = frac # name = f'{model}_epoch_{convergence_epoch[model]:02d}' frac = (np.mean(model_dict[model][selection]) / full) matrix[-1, -1] = frac plot_heatmap(matrix, r'\textbf{Epochs}', r'\textbf{Images}', title=title, annot=True, ax=ax, cbar=False, cmap='YlOrRd', percent=False, fmt='.0%', vmin=0, vmax=1, yticklabels=imgs + ['All'], xticklabels=epochs + ['Convergence'], alpha=0.8)
def plot_over_epoch(models): model_dict = {} conn = get_connection() epochs = (0, 5, 10, 15) for model in models: names = [] for epoch in epochs: names.append(f'{model}_epoch_{epoch:02d}') model_dict[model] = load_scores(conn, names, benchmarks=benchmarks) model_dict[f'{model}_epoch_00'] = load_scores(conn, ['CORnet-S_random'], benchmarks) benchmarks_labels = ['V1', 'V2', 'V4', 'IT', 'Behavior', 'Imagenet'] for i in range(6): data = {} for model in models: data[model] = [] for epoch in epochs: data[model].append(model_dict[model][f'{model}_epoch_{epoch:02d}'][i]) # data['CORnet-S'] = [0] * 3 + [model_dict['CORnet-S']['CORnet-S'][i]] plot_data_base(data, f'{benchmarks_labels[i]} Benchmark over epochs', epochs, 'Score over epochs', 'Score')
def image_scores_single(model, imgs, selection=[], ax=None): names = [] conn = get_connection() for img in imgs: name = f'{model}_img{img}' names.append(name) # if model == 'CORnet-S_cluster2_v2_IT_trconv3_bi': # model = f'{model}_seed42' # names.append(f'{model}_epoch_{convergence_epoch[model]}') names.append('CORnet-S_full') # model_dict = load_scores(conn, names, benchmarks) model_dict = load_error_bared(conn, names, benchmarks, convergence=True) full = model_dict[model] benchmarks_labels = ['V1', 'V2', 'V4', 'IT', 'Behavior', 'Mean'] data = {} for i in range(len(benchmarks) - 1): if i in selection: data[benchmarks_labels[i]] = [] for j in imgs: name1 = f'{model}_img{j}' frac = (np.mean(model_dict[name1][i]) / full[i]) * 100 data[benchmarks_labels[i]].append(frac) frac = (np.mean(model_dict[model][i]) / full[i]) * 100 data[benchmarks_labels[i]].append(frac) data[benchmarks_labels[-1]] = [] for j in imgs: name1 = f'{model}_img{j}' frac = (np.mean(model_dict[name1][selection]) / np.mean(full[selection])) * 100 data[benchmarks_labels[-1]].append(frac) frac = (np.mean(model_dict[model][selection]) / np.mean(full[selection])) * 100 data[benchmarks_labels[-1]].append(frac) imgs.append(1280000) plot_data_base(data, '', r'\textbf{Images} [Million]', r'\textbf{Score} [\% of standard training]', x_values=imgs, x_ticks=[100, 1000, 10000, 100000, 1280000], x_labels=['100', '1k', '10k', '100k', '1.3M'], million_base=True, palette=grey_palette[:len(benchmarks_labels) - 1] + [blue_palette[0]], alpha=0.5, use_xticks=True, percent=True, log=True, annotate=True, legend=False, annotate_pos=3, ax=ax)
def plot_num_params_images(imagenet=False, entry_models=[], all_labels=[], images=[], convergence=False, ax=None, selection=[], log=False, layer_random=layer_random): conn = get_connection() full = np.mean(get_full(conn, convergence)[selection]) data2 = {} labels = [] params = {} for entry_model, name in itertools.chain( [(layer_random, 'Kaiming Normal + Downstream Training (KN+DT)')], zip(entry_models, all_labels)): short = name.split('(')[1][:-1] for img in images: name_epoch = f'{short} {img} Imgs' data2[name_epoch] = [] params[name_epoch] = [] data2[f'{short} Full'] = [] params[f'{short} Full'] = [] mod_params = get_model_params(entry_model.keys()) names = [] for model in entry_model.keys(): if model == "CORnet-S_random": names.append(model) else: names.append(f'{model}_epoch_{convergence_epoch[model]}') model = model.split('_seed42')[0] for img in images: model_img = f'{model}_img{img}' conv = convergence_images[ model_img] if model_img in convergence_images else 20 names.append(f'{model_img}_epoch_{conv:02d}') model_dict = load_scores(conn, names, benchmarks) for model in names: percent = (np.mean(model_dict[model][selection]) / full) * 100 if 'img' not in model: name_epoch = f'{short} Full' base_model = model.partition('_epoch')[0] data2[name_epoch].append(percent) params[name_epoch].append(mod_params[base_model]) else: img = model.split('_')[-3].partition('g')[2] base_model = model.partition('_img')[0] name_epoch = f'{short} {img} Imgs' data2[name_epoch].append(percent) params[name_epoch].append(mod_params[base_model]) labels = labels + [f'{ep} Images' for ep in images] + ['Convergence'] if imagenet: title = f'Imagenet score vs number of parameter' y = r'Imagenet performance [% of standard training]' else: title = f'Brain-Score Benchmark mean(V4, IT, Behavior) vs number of parameter' if len(selection) == 3: y = r"Brain Predictivity [% of standard training]" else: y = r"Brain Predictivity [% of standard training]" col = grey_palette[:len(images) + 1] + blue_palette[:len( images) + 1] + green_palette[:len(images) + 1] + grey_palette[:len(images) + 1] plot_data_double(data2, {}, '', x_name='Number of trained parameters [Million]', x_labels=[], y_name=y, x_ticks=params, pal=col, data_labels=labels, ylim=[0, 100], x_ticks_2=[], percent=True, ax=ax, million=True, annotate_pos=0, log=log)
def score_over_layers_avg(all_models, random, all_labels=[], imagenet=False, convergence=False, ax=None, selection=[]): conn = get_connection() data = {} err2 = {} full = 0 names = [] for models in all_models: names.extend(models.keys()) names.extend(random.keys()) model_dict = load_error_bared(conn, names, benchmarks, convergence=convergence) data2 = {} data2['Score'] = [] err2['Score'] = [] layers2 = [] for model, layer in random.items(): layers2.append(layer_best_2[layer]) if model == 'CORnet-S_full': full = np.mean(model_dict[model][selection]) full_err = (np.mean(model_dict[model][6:][selection]) / full) * 100 data2['Score'].append(100) err2['Score'].append(full_err) else: percent = (np.mean(model_dict[model][selection]) / full) * 100 percent_error = (np.mean(model_dict[model][6:][selection]) / full) * 100 data2['Score'].append(percent) err2['Score'].append(percent_error) x_ticks = {} labels = {} err = {} for models, name in zip(all_models, all_labels): data[name] = [] err[name] = [] layers = [] for model, layer in models.items(): if convergence and model in convergence_epoch: postfix = f'_epoch_{convergence_epoch[model]:02d}' else: postfix = f'_epoch_06' layers.append(layer_best_2[layer]) if model == 'CORnet-S_full': full = np.mean(model_dict[model][selection]) data[name].append(100) else: percent = (np.mean(model_dict[model][selection]) / full) * 100 data[name].append(percent) full_err = (np.mean(model_dict[model][6:][selection]) / full) * 100 err[name].append(full_err) x_ticks[name] = layers # short = name.split('(')[1][:-1] # labels[name] = [f'{value.split(".")[0]}_{short}' for value in models.values()] labels[name] = [name] if imagenet: title = f'Imagenet over layers' y = r"Imagenet}[% of standard training]" else: title = f'Brain-Score Benchmark mean(V4, IT, Behavior) over layers' if len(selection) == 3: y = r"Brain Predictivity [% of standard training]" else: y = r"Brain Predictivity [% of standard training]" plot_data_double(data, data2, '', err=err, err2=err2, x_name='Number of trained layers', y_name=y, x_ticks=x_ticks, x_ticks_2=layers2, percent=True, annotate_pos=0, pal=my_palette, ax=ax)
def plot_performance(imagenet=True, entry_models=[best_brain_avg], all_labels=[], convergence=False, ax=None, selection=[], log=False): conn = get_connection() names = [] for model in random_scores.keys(): if convergence and model in convergence_epoch: postfix = f'_epoch_{convergence_epoch[model]:02d}' else: postfix = f'_epoch_06' if model != "CORnet-S_random": names.append(f'{model}{postfix}') else: names.append(model) performance = load_model_parameter(conn) model_dict = load_scores(conn, names, benchmarks) time2 = [] data2 = {'Score': []} for model, layer in random_scores.items(): if model == "CORnet-S_random": postfix = '' elif convergence and model in convergence_epoch: postfix = f'_epoch_{convergence_epoch[model]:02d}' else: postfix = f'_epoch_06' high = np.mean(model_dict[ f'CORnet-S_full_epoch_{convergence_epoch["CORnet-S_full"]}'] [selection]) perc = (np.mean(model_dict[f'{model}{postfix}'][selection]) / high) * 100 if layer in performance: data2['Score'].append(perc) time2.append(performance[layer]) data = {} time = {} labels = {} for entry_model, name in zip(entry_models, all_labels): names = [] for model in entry_model.keys(): if convergence and model in convergence_epoch: postfix = f'_epoch_{convergence_epoch[model]:02d}' else: postfix = f'_epoch_06' names.append(f'{model}{postfix}') model_dict = load_scores(conn, names, benchmarks) time[name] = [] data[name] = [] for model, layer in entry_model.items(): if convergence and model in convergence_epoch: postfix = f'_epoch_{convergence_epoch[model]:02d}' else: postfix = f'_epoch_06' perc = (np.mean(model_dict[f'{model}{postfix}'][selection]) / high) * 100 if layer in performance: data[name].append(perc) time[name].append(performance[layer]) short = name.split('(')[1][:-1] labels[name] = [ f'{value.split(".")[0]}_{short}' for value in entry_model.values() ] if imagenet: title = f'Imagenet score vs training time' y = r'Imagenet performance [% of standard training]' else: title = f'Brain-Score Benchmark mean(V4, IT, Behavior) vs training time' if len(selection) == 3: y = r"Brain Predictivity) [% of standard training]" else: y = r"Brain Predictivity [% of standard training]" plot_data_double(data, data2, '', x_name='Training time [Milliseconds/Epoch]', x_labels=[], y_name=y, x_ticks=time, x_ticks_2=time2, percent=True, data_labels=labels, ax=ax, log=log)
def plot_num_params(imagenet=False, entry_models=[], all_labels=[], convergence=False, ax=None, selection=[], log=False, layer_random=layer_random, pal=None, percent=True, ylim=None): conn = get_connection() full_score = get_full(conn, convergence) full = np.mean(full_score[selection]) names = [] data = {'Score': []} err = {'Score': []} model_dict = load_error_bared(conn, layer_random.keys(), benchmarks, convergence=convergence) for model, layer in layer_random.items(): if percent: frac = (np.mean(model_dict[model][selection]) / full) * 100 percent_err = (np.mean(model_dict[model][6:][selection]) / full) * 100 else: frac = np.mean(model_dict[model][selection]) percent_err = np.mean(model_dict[model][6:][selection]) print(f'MOdel {model} has score {frac}') data['Score'].append(frac) err['Score'].append(percent_err) data2 = {} err2 = {} labels = {} params = {} for entry_model, name in zip(entry_models, all_labels): data2[name] = [] params[name] = [] err2[name] = [] for model in entry_model.keys(): print(model) if model.endswith('BF'): model = model.replace('_BF', '') if model.endswith('seed42'): model = model.replace('_seed42', '') if model == "CORnet-S_random": params[name].append(0) else: params[name].append(get_params(model)) model_dict = load_error_bared(conn, entry_model.keys(), benchmarks) for model in entry_model.keys(): if percent: frac = (np.mean(model_dict[model][selection]) / full) * 100 percent_err = (np.mean(model_dict[model][6:][selection]) / full) * 100 else: frac = np.mean(model_dict[model][selection]) percent_err = np.mean(model_dict[model][6:][selection]) print(f'MOdel {model} has score {frac}') data2[name].append(frac) err2[name].append(percent_err) if '(' in name: short = name.split('(')[1][:-1] else: short = name if len(entry_model) == 1: labels[name] = [short, f'{short} Transfer'] else: labels[name] = [short] params2 = [] for model in layer_random.keys(): if model.endswith('BF'): model = model.replace('_BF', '') if model == "CORnet-S_random": params2.append(0) else: params2.append(get_params(model)) if imagenet: title = f'Imagenet score vs number of parameter' y = r'\textbf{Imagenet performance}' else: title = f'Brain-Score Benchmark mean(V4, IT, Behavior) vs number of parameter' if len(selection) == 3: y = r"\textbf{Brain Predictivity}" else: y = r"\textbf{Brain Predictivity}" if percent: y = f'{y} [\% of standard training]' if pal is None: pal = blue_palette plot_data_double(data2, data, '', err=err2, err2=err, x_name=r'\textbf{Number of trained parameters} [Million]', x_labels=None, ylim=ylim, y_name=y, x_ticks=params, pal=pal, percent=percent, x_ticks_2=params2, data_labels=labels, ax=ax, million=True, log=log, annotate_pos=0)
def plot_num_params_epochs(imagenet=False, entry_models=[], all_labels=[], epochs=[], convergence=False, ax=None, selection=[], log=False, layer_random=layer_random): conn = get_connection() full = np.mean(get_full(conn, convergence)[selection]) data2 = {} labels = [] params = {} for entry_model, name in itertools.chain( [(layer_random, 'Kaiming Normal + Downstream Training (KN+DT)')], zip(entry_models, all_labels)): short = name.split('(')[1][:-1] for epoch in epochs: name_epoch = f'{short} Epoch {epoch:02d}' data2[name_epoch] = [] params[name_epoch] = [] data2[f'{short} Convergence'] = [] params[f'{short} Convergence'] = [] mod_params = get_model_params(entry_model.keys()) names = [] for model in entry_model.keys(): if model == "CORnet-S_random": names.append(model) else: conv = convergence_epoch[ model] if model in convergence_epoch else 100 for epoch in epochs: if epoch < conv: names.append(f'{model}_epoch_{epoch:02d}') if convergence and model in convergence_epoch: names.append(f'{model}_epoch_{conv:02d}') model_dict = load_scores(conn, names, benchmarks) for model in names: epoch = model.split('_')[-1] base_model = model.partition('_epoch')[0] percent = (np.mean(model_dict[model][selection]) / full) * 100 if int(epoch) == convergence_epoch[base_model]: name_epoch = f'{short} Convergence' data2[name_epoch].append(percent) params[name_epoch].append(mod_params[base_model]) if int(epoch) in epochs: name_epoch = f'{short} Epoch {epoch}' data2[name_epoch].append(percent) params[name_epoch].append(mod_params[base_model]) labels = labels + [f'Epoch {ep}' for ep in epochs] + ['Convergence'] if imagenet: title = f'Imagenet score vs number of parameter' y = r'Imagenet performance [% of standard training]' else: title = f'Brain Predictivity vs number of parameter' if len(selection) == 3: y = r"Brain Predictivity [% of standard training]" else: y = r"Brain Predictivity [% of standard training]" col = grey_palette[:len(epochs) + 1] + blue_palette[:len( epochs) + 1] + green_palette[:len(epochs) + 1] + grey_palette[:len(epochs) + 1] plot_data_double( data2, {}, '', x_name='Number of trained parameters [Million]', x_labels=[], y_name=y, data_labels=labels, x_ticks=params, pal=col, x_ticks_2=[], percent=True, ax=ax, million=True, ylim=[0, 100], annotate_pos=0, log=log, )
def image_epoch_score(models, imgs, epochs, selection=[], axes=None, percent=True, make_trillions=False, with_weights=True): names = [] conn = get_connection() params = {} data = {} for model, label in models.items(): data[label] = [] params[label] = [] if model == 'CORnet-S_cluster2_v2_IT_trconv3_bi': model1 = model else: model1 = model for img in imgs: name = f'{model}_img{img}' for epoch in epochs: if epoch % 1 == 0: names.append(f'{name}_epoch_{epoch:02d}') else: names.append(f'{name}_epoch_{epoch:.1f}') if name in convergence_images: names.append(f'{name}_epoch_{convergence_images[name]}') names.append(f'{model1}_epoch_{convergence_epoch[model1]:02d}') if with_weights: parameter = get_model_params(models, False) else: parameter = {x: 1 for x in models} names.append('CORnet-S_full_epoch_43') model_dict = load_scores(conn, names, benchmarks) full = np.mean(model_dict['CORnet-S_full_epoch_43'][selection]) high_x = 0 high_y = 0 val = 0 for model in names: if percent: frac = (np.mean(model_dict[model][selection]) / full) * 100 else: frac = np.mean(model_dict[model][selection]) if frac > 0.0: if 'img' not in model: base_model = model.partition('_epoch')[0] epoch = float(model.partition('_epoch_')[2]) data[models[base_model]].append(frac) score = (1280000 * epoch * (parameter[base_model] / 1000000) ) # print( f'Model {base_model} in epoch {epoch} with full imagenet set ' f'leads to score {score} with brain score {frac}') params[models[base_model]].append(score) else: base_model = model.partition('_img')[0] imgs = int(model.partition('_img')[2].partition('_')[0]) epoch = float( model.partition('_img')[2].partition('_epoch_')[2]) score = (imgs * epoch * (parameter[base_model] / 1000000) ) # (parameter[base_model] / 1000000) * data[models[base_model]].append(frac) params[models[base_model]].append(score) print( f'Model {base_model} in epoch {epoch} with {imgs} images ' f'leads to score {score} with brain score {frac}') if percent > high_y: high_y = percent if len(selection) == 3: y = r"\textbf{Brain Predictivity}" else: y = r"\textbf{Brain Predictivity}[\% of standard training]" # [\% of standard training] for i, ax in enumerate(axes): zero_indices = { key: np.array([tick == 0 for tick in xticks]) for key, xticks in params.items() } if i == 0: # axis plotting the x=0 value ax_data = { key: np.array(values)[zero_indices[key]].tolist() for key, values in data.items() } xticks = { key: np.array(values)[zero_indices[key]].tolist() for key, values in params.items() } xticklabels = np.array([0]) ylabel = y else: # axis plotting everything x>0 ax_data = { key: np.array(values)[~zero_indices[key]].tolist() for key, values in data.items() } xticks = { key: np.array(values)[~zero_indices[key]].tolist() for key, values in params.items() } # when make_trillions==True, this should actually be *10^12, but due to downstream hacks we leave it at ^6 xticklabels = np.array([.001, .01, .1, 1, 10, 100, 1000]) * pow( 10, 6) ax.spines['left'].set_visible(False) ylabel = '' kwargs = dict(trillion=True) if make_trillions else dict( trillion=True, million_base=True) plot_data_double(ax_data, {}, '', x_name='', x_labels=xticklabels, scatter=True, percent=percent, alpha=0.8, ylim=[0, 100], y_name=ylabel, x_ticks=xticks, pal=[ '#2CB8B8', '#186363', '#ABB2B9', '#ABB2B9', '#ABB2B9', '#259C9C', '#36E3E3', '#9AC3C3' ], log=True, x_ticks_2={}, ax=ax, **kwargs, annotate_pos=0) # adopted from https://stackoverflow.com/a/32186074/2225200 d = .015 # how big to make the diagonal lines in axes coordinates kwargs = dict(transform=ax.transAxes, color='#dedede', clip_on=False) if i == 0: m = 1 / .05 ax.plot((1 - d * m, 1 + d * m), (-d, +d), **kwargs) else: kwargs.update(transform=ax.transAxes) ax.plot((-d, +d), (-d, +d), **kwargs) # remove yticks. We can't `ax.yaxis.set_visible(False)` altogether since that would also remove the grid for tic in ax.yaxis.get_major_ticks(): tic.tick1On = tic.tick2On = False ax.set_yticklabels([]) axes[0].set_ylim(axes[1].get_ylim())