def get_quantiles_helpers(architecture: Architecture, dataset: Dataset, dataset_size: int) -> typing.Dict: assert architecture.is_trained quantiles_helpers = dict() min_max_vals = dict() dataset = get_sample_dataset( epsilon=0.0, noise=0.0, adv=False, archi=architecture, dataset_size=dataset_size, succ_adv=False, dataset=dataset, compute_graph=False, train=False, ) logger.info(f"Got dataset of size {len(dataset)}") size_for_min_max = 10 # Checking min_max for i, line in enumerate(dataset[:size_for_min_max]): logger.info(f"Computing min/max sample {i}/{size_for_min_max}") graph = Graph.from_architecture_and_data_point( architecture=architecture, x=line.x) for key in graph._edge_dict: layer_matrix = graph._edge_dict[key] min_val, max_val = np.min(layer_matrix.data), np.max( layer_matrix.data) old_min, old_max = min_max_vals.get(key, (np.inf, 0)) min_val = min([min_val, old_min]) max_val = max([max_val, old_max]) min_max_vals[key] = (min_val, max_val) del graph logger.info(f"Min-max values are {min_max_vals}") # Creating quantile helpers for i, line in enumerate(dataset): logger.info(f"Computing histograms for quantiles {i}/{len(dataset)}") graph = Graph.from_architecture_and_data_point( architecture=architecture, x=line.x) for key in graph._edge_dict: layer_matrix = graph._edge_dict[key] # logger.info(f"Line {line.sample_id}: {key}: {layer_matrix.shape}") if key not in quantiles_helpers: min_val, max_val = min_max_vals[key] quantiles_helpers[key] = HistogramQuantile( min_value=0.9 * min_val, max_value=1.1 * max_val, precision=int(1e6)) helper: HistogramQuantile = quantiles_helpers[key] helper.add_values(layer_matrix.data) del graph return quantiles_helpers
def test_simple_resnet_graph(): simple_archi: Architecture = Architecture( preprocess=lambda x: x, layers=[ LinearLayer(4, 4), LinearLayer(4, 4), LinearLayer(4, 4), LinearLayer(4, 10), SoftMaxLayer(), ], layer_links=[(-1, 0), (0, 1), (1, 2), (1, 3), (2, 3), (3, 4)], ) simple_archi.build_matrices() simple_example = torch.ones(4) graph = Graph.from_architecture_and_data_point(simple_archi, simple_example) adjacency_matrix = graph.get_adjacency_matrix().todense() assert np.shape(adjacency_matrix) == (26, 26) assert len(graph.get_edge_list()) == 128 assert len(np.where(adjacency_matrix > 0)[0]) == 128 * 2 print(graph.get_edge_list()) print(simple_archi.get_pre_softmax_idx())
def test_matshapes(architecture, shape): simple_example = torch.randn(shape) architecture.forward(simple_example) architecture.build_matrices() graph = Graph.from_architecture_and_data_point(architecture, simple_example) conv_layers = list() for layer_idx, layer in enumerate(architecture.layers): if isinstance(layer, ConvLayer): conv_layers.append(layer_idx) source_conv_layers = list() for layer_idx in conv_layers: source_conv_layers.append([u for u, v in architecture.layer_links if v == layer_idx][0]) shapes = graph._get_shapes() ret = "[" for u, v in zip(source_conv_layers, conv_layers): shape0 = shapes[u] shape1 = shapes[v] ret += f"[{shape1}, {shape0}]," ret += "]" print(ret)
def test_simple_cnn_multi_channels(): simple_archi = Architecture( preprocess=lambda x: x, layers=[ # 2 input channels # 3 output channels ConvLayer(2, 3, 2, input_shape=(3, 4)), LinearLayer(18, 1), ], ) simple_archi.build_matrices() simple_example = torch.tensor( [ [ [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]], [[2, 4, 6, 8], [10, 12, 14, 16], [18, 20, 22, 24]], ] ] ) for param in simple_archi.parameters(): print(f"Kernel size is {list(param.shape)}") m = simple_archi.get_graph_values(simple_example) # Shape should be 6*3 out_channels = 18 x 12*2 in_channels = 24 assert np.shape(m[(-1, 0)]) == (18, 24) assert np.shape(m[(0, 1)]) == (1, 18) graph = Graph.from_architecture_and_data_point(simple_archi, simple_example) adjacency_matrix = graph.get_adjacency_matrix() assert np.shape(adjacency_matrix) == (18 + 24 + 1, 18 + 24 + 1)
def test_svhn_graph(): simple_example = torch.ones((3, 32, 32)) * 0.2 for param in svhn_cnn_simple.parameters(): param.data = torch.ones_like(param.data) * 0.5 svhn_cnn_simple.forward(simple_example) svhn_cnn_simple.build_matrices() graph = Graph.from_architecture_and_data_point(svhn_cnn_simple, simple_example) adjacency_matrix = graph.get_adjacency_matrix() assert np.shape(adjacency_matrix) == (11838, 11838) assert np.linalg.norm(adjacency_matrix.todense()) == 5798210602234079.0
def test_mnist_graph(): simple_example = torch.randn((28, 28)) mnist_mlp.build_matrices() graph = Graph.from_architecture_and_data_point(mnist_mlp, simple_example) adjacency_matrix = graph.get_adjacency_matrix().todense() assert np.shape(adjacency_matrix) == (1550, 1550) assert len(graph.get_edge_list()) == 522560 assert len(np.where(adjacency_matrix > 0)[0]) == 522560 * 2 print(graph.get_edge_list())
def test_sliced_wasserstein_gram_matrix(benchmark): simple_archi: Architecture = Architecture( preprocess=lambda x: x, layers=[LinearLayer(4, 3), SoftMaxLayer()] ) simple_archi.build_matrices() embeddings = list() for val in np.random.randint(1, 100, 100): idx = np.random.randint(0, 4) z = np.zeros(4) z[idx] = val ex = torch.from_numpy(z) g = Graph.from_architecture_and_data_point(simple_archi, ex) dgm = compute_dgm_from_graph(g) embeddings.append(dgm) def compute_matrix(): return get_gram_matrix( kernel_type=KernelType.SlicedWasserstein, embeddings_in=embeddings, embeddings_out=embeddings, params=[{"M": 50, "sigma": 0.5}], )[0] legacy_matrix = get_gram_matrix_legacy( kernel_type=KernelType.SlicedWasserstein, embeddings_in=embeddings, embeddings_out=embeddings, params={"M": 50, "sigma": 0.5}, ) print(legacy_matrix) matrix = benchmark(compute_matrix) print(matrix) assert np.isclose(np.linalg.norm(matrix - legacy_matrix), 0.0)
def get_embedding( embedding_type: str, line: DatasetLine, architecture: Architecture, quantiles_helpers_for_sigmoid=None ) -> Embedding: time_taken = dict() if line.graph is None: start = time.time() graph = Graph.from_architecture_and_data_point( architecture=architecture, x=line.x.double() ) time_taken["graph"] = time.time()-start else: graph = line.graph if quantiles_helpers_for_sigmoid is not None: start = time.time() graph.sigmoidize(quantiles_helpers=quantiles_helpers_for_sigmoid) time_taken["sigmoidize"] = time.time() - start if embedding_type == EmbeddingType.PersistentDiagram: start = time.time() dgm = compute_dgm_from_graph(graph) del graph time_taken[f"E_{embedding_type}"] = time.time() - start return Embedding(value=dgm, time_taken=time_taken) elif embedding_type == EmbeddingType.RawGraph: start = time.time() mat = to_sparse_vector(graph.get_adjacency_matrix()) del graph time_taken[f"E_{embedding_type}"] = time.time() - start return Embedding(value=mat, time_taken=time_taken) else: raise NotImplementedError(embedding_type)
) from tda.graph import Graph from tda.models import Architecture from tda.models.architectures import LinearLayer, SoftMaxLayer simple_archi: Architecture = Architecture( preprocess=lambda x: x, layers=[LinearLayer(4, 3), LinearLayer(3, 2), LinearLayer(2, 10), SoftMaxLayer()], ) ex1 = torch.ones(4) * 42 ex2 = torch.ones(4) * 37 simple_archi.build_matrices() g1 = Graph.from_architecture_and_data_point(simple_archi, ex1) g2 = Graph.from_architecture_and_data_point(simple_archi, ex2) dgm1 = compute_dgm_from_graph(g1, astuple=False) dgm2 = compute_dgm_from_graph(g2, astuple=False) dgm1_tuple = compute_dgm_from_graph(g1, astuple=True) dgm2_tuple = compute_dgm_from_graph(g2, astuple=True) def test_simple_edge_list(): edge_list = [((0, 1), np.float64(3.0)), ((0, 2), np.float64(2.0))] ret = compute_dgm_from_edges(edge_list)
def get_sample_dataset( epsilon: float, noise: float, adv: bool, dataset: Dataset, train: bool, succ_adv: bool, archi: Architecture = mnist_mlp, dataset_size: int = 100, attack_type: str = "FGSM", attack_backend: str = AttackBackend.FOOLBOX, num_iter: int = 10, offset: int = 0, per_class: bool = False, compute_graph: bool = False, transfered_attacks: bool = False, ) -> typing.List[DatasetLine]: logger.info(f"Using source dataset {dataset.name}") logger.info(f"Checking that the received architecture has been trained") assert archi.is_trained logger.info(f"OK ! Architecture is ready") logger.info( f"I am going to generate a dataset of {dataset_size} points...") if adv: logger.info( f"Only successful adversaries ? {'yes' if succ_adv else 'no'}") logger.info(f"Which attack ? {attack_type}") logger.info(f"Which backend ? {attack_backend}") else: logger.info("This dataset will be non-adversarial !") if transfered_attacks: logger.info( f"Loading the architecture to generate adversaries with transferred attacks with {archi.epochs} epochs" ) archi = architecture = get_deep_model( num_epochs=archi.epochs, dataset=dataset, architecture=archi, train_noise=archi.train_noise, ) if archi.epochs % 10 == 0: archi.epochs += 1 list_locals = locals() logger.info(f"locals = {list_locals}") source_dataset_path = get_my_path(list_locals) if os.path.exists(source_dataset_path): source_dataset = torch.load(source_dataset_path) source_dataset_size = len(source_dataset) logger.info( f"Successfully loaded dataset of trsf attacks (len {source_dataset_size})" ) archi.epochs -= 1 current_sample_id = 0 else: source_dataset = (dataset.train_dataset if train else dataset.test_and_val_dataset) source_dataset_size = len(source_dataset) current_sample_id = offset else: source_dataset = (dataset.train_dataset if train else dataset.test_and_val_dataset) source_dataset_size = len(source_dataset) current_sample_id = offset final_dataset = list() if dataset.name in ["tinyimagenet"]: per_class_nb_samples = np.repeat(0, 200) elif dataset.name in ["cifar100"]: per_class_nb_samples = np.repeat(0, 100) else: per_class_nb_samples = np.repeat(0, 10) #current_sample_id = offset dataset_done = False batch_size = 32 while not dataset_done and current_sample_id < source_dataset_size: samples = None processed_samples = None y_pred = None while processed_samples is None and current_sample_id < source_dataset_size: if False: #transfered_attacks: samples = ( source_dataset["x"][current_sample_id:current_sample_id + batch_size], source_dataset["y"][current_sample_id:current_sample_id + batch_size], ) if adv: processed_samples = ( source_dataset["x_adv"] [current_sample_id:current_sample_id + batch_size], source_dataset["y"] [current_sample_id:current_sample_id + batch_size], ) else: processed_samples = samples else: # Fetching a batch of samples and concatenating them batch = source_dataset[current_sample_id:current_sample_id + batch_size] if isinstance(batch[0], DatasetLine): x = torch.cat([torch.unsqueeze(s.x, 0) for s in batch], 0).to(device) y = np.array([s.y for s in batch]) logger.info(f"shape of x = {x.shape}") else: x = torch.cat([torch.unsqueeze(s[0], 0) for s in batch], 0).to(device) y = np.array([s[1] for s in batch]) samples = (x, y) # Calling process_sample on the batch # TODO: Ensure process_sample handles batched examples processed_samples = process_sample( sample=samples, adversarial=adv, noise=noise, epsilon=epsilon, model=archi, attack_type=attack_type, num_iter=num_iter, attack_backend=attack_backend, ) # Increasing current_sample_id current_sample_id += batch_size assert (samples[1] == processed_samples[1]).all() y_pred = archi(processed_samples[0]).argmax(dim=-1).cpu().numpy() if adv and succ_adv: # Check where the attack was successful valid_attacks = np.where(samples[1] != y_pred)[0] logger.debug( f"Attack succeeded on {len(valid_attacks)} points over {len(samples[1])}" ) if len(valid_attacks) == 0: processed_samples = None else: processed_samples = ( processed_samples[0][valid_attacks], processed_samples[1][valid_attacks], ) samples = ( samples[0][valid_attacks], samples[1][valid_attacks], ) # If the while loop did not return any samples, let's stop here if processed_samples is None: break # Compute the norms on the batch l2_norms = (torch.norm( (processed_samples[0].double() - samples[0].double()).flatten(1), p=2, dim=1, ).cpu().detach().numpy()) linf_norms = (torch.norm( (processed_samples[0].double() - samples[0].double()).flatten(1), p=float("inf"), dim=1, ).cpu().detach().numpy()) # Update the counter per class for clazz in processed_samples[1]: per_class_nb_samples[clazz] += 1 # Unbatching and return DatasetLine # TODO: see if we can avoid unbatching for i in range(len(processed_samples[1])): x = torch.unsqueeze(processed_samples[0][i], 0).double() # (OPT) Compute the graph graph = (Graph.from_architecture_and_data_point( architecture=archi, x=x) if compute_graph else None) # Add the line to the dataset final_dataset.append( DatasetLine( graph=graph, x=x, y=processed_samples[1][i], y_pred=y_pred[i], y_adv=adv, l2_norm=l2_norms[i], linf_norm=linf_norms[i], sample_id=current_sample_id, )) # Are we done yet ? if not per_class: # We are done if we have enough points in the dataset dataset_done = len(final_dataset) >= dataset_size else: # We are done if all classes have enough points dataset_done = all( np.asarray(per_class_nb_samples) >= dataset_size) if dataset_done: break logger.info(f"Compputed {len(final_dataset)}/{dataset_size} samples.") if len(final_dataset) < dataset_size: logger.warn( f"I was only able to generate {len(final_dataset)} points even if {dataset_size} was requested. " f"This is probably a lack of adversarial points.") return final_dataset