def get_quantiles_helpers(architecture: Architecture, dataset: Dataset,
dataset_size: int) -> typing.Dict:
assert architecture.is_trained
quantiles_helpers = dict()
min_max_vals = dict()
dataset = get_sample_dataset(
epsilon=0.0,
noise=0.0,
adv=False,
archi=architecture,
dataset_size=dataset_size,
succ_adv=False,
dataset=dataset,
compute_graph=False,
train=False,
)
logger.info(f"Got dataset of size {len(dataset)}")
size_for_min_max = 10
# Checking min_max
for i, line in enumerate(dataset[:size_for_min_max]):
logger.info(f"Computing min/max sample {i}/{size_for_min_max}")
graph = Graph.from_architecture_and_data_point(
architecture=architecture, x=line.x)
for key in graph._edge_dict:
layer_matrix = graph._edge_dict[key]
min_val, max_val = np.min(layer_matrix.data), np.max(
layer_matrix.data)
old_min, old_max = min_max_vals.get(key, (np.inf, 0))
min_val = min([min_val, old_min])
max_val = max([max_val, old_max])
min_max_vals[key] = (min_val, max_val)
del graph
logger.info(f"Min-max values are {min_max_vals}")
# Creating quantile helpers
for i, line in enumerate(dataset):
logger.info(f"Computing histograms for quantiles {i}/{len(dataset)}")
graph = Graph.from_architecture_and_data_point(
architecture=architecture, x=line.x)
for key in graph._edge_dict:
layer_matrix = graph._edge_dict[key]
# logger.info(f"Line {line.sample_id}: {key}: {layer_matrix.shape}")
if key not in quantiles_helpers:
min_val, max_val = min_max_vals[key]
quantiles_helpers[key] = HistogramQuantile(
min_value=0.9 * min_val,
max_value=1.1 * max_val,
precision=int(1e6))
helper: HistogramQuantile = quantiles_helpers[key]
helper.add_values(layer_matrix.data)
del graph
return quantiles_helpers
def test_simple_resnet_graph():
simple_archi: Architecture = Architecture(
preprocess=lambda x: x,
layers=[
LinearLayer(4, 4),
LinearLayer(4, 4),
LinearLayer(4, 4),
LinearLayer(4, 10),
SoftMaxLayer(),
],
layer_links=[(-1, 0), (0, 1), (1, 2), (1, 3), (2, 3), (3, 4)],
)
simple_archi.build_matrices()
simple_example = torch.ones(4)
graph = Graph.from_architecture_and_data_point(simple_archi, simple_example)
adjacency_matrix = graph.get_adjacency_matrix().todense()
assert np.shape(adjacency_matrix) == (26, 26)
assert len(graph.get_edge_list()) == 128
assert len(np.where(adjacency_matrix > 0)[0]) == 128 * 2
print(graph.get_edge_list())
print(simple_archi.get_pre_softmax_idx())
def test_matshapes(architecture, shape):
simple_example = torch.randn(shape)
architecture.forward(simple_example)
architecture.build_matrices()
graph = Graph.from_architecture_and_data_point(architecture, simple_example)
conv_layers = list()
for layer_idx, layer in enumerate(architecture.layers):
if isinstance(layer, ConvLayer):
conv_layers.append(layer_idx)
source_conv_layers = list()
for layer_idx in conv_layers:
source_conv_layers.append([u for u, v in architecture.layer_links if v == layer_idx][0])
shapes = graph._get_shapes()
ret = "["
for u, v in zip(source_conv_layers, conv_layers):
shape0 = shapes[u]
shape1 = shapes[v]
ret += f"[{shape1}, {shape0}],"
ret += "]"
print(ret)
def test_simple_cnn_multi_channels():
simple_archi = Architecture(
preprocess=lambda x: x,
layers=[
# 2 input channels
# 3 output channels
ConvLayer(2, 3, 2, input_shape=(3, 4)),
LinearLayer(18, 1),
],
)
simple_archi.build_matrices()
simple_example = torch.tensor(
[
[
[[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]],
[[2, 4, 6, 8], [10, 12, 14, 16], [18, 20, 22, 24]],
]
]
)
for param in simple_archi.parameters():
print(f"Kernel size is {list(param.shape)}")
m = simple_archi.get_graph_values(simple_example)
# Shape should be 6*3 out_channels = 18 x 12*2 in_channels = 24
assert np.shape(m[(-1, 0)]) == (18, 24)
assert np.shape(m[(0, 1)]) == (1, 18)
graph = Graph.from_architecture_and_data_point(simple_archi, simple_example)
adjacency_matrix = graph.get_adjacency_matrix()
assert np.shape(adjacency_matrix) == (18 + 24 + 1, 18 + 24 + 1)
def test_svhn_graph():
simple_example = torch.ones((3, 32, 32)) * 0.2
for param in svhn_cnn_simple.parameters():
param.data = torch.ones_like(param.data) * 0.5
svhn_cnn_simple.forward(simple_example)
svhn_cnn_simple.build_matrices()
graph = Graph.from_architecture_and_data_point(svhn_cnn_simple, simple_example)
adjacency_matrix = graph.get_adjacency_matrix()
assert np.shape(adjacency_matrix) == (11838, 11838)
assert np.linalg.norm(adjacency_matrix.todense()) == 5798210602234079.0
def test_mnist_graph():
simple_example = torch.randn((28, 28))
mnist_mlp.build_matrices()
graph = Graph.from_architecture_and_data_point(mnist_mlp, simple_example)
adjacency_matrix = graph.get_adjacency_matrix().todense()
assert np.shape(adjacency_matrix) == (1550, 1550)
assert len(graph.get_edge_list()) == 522560
assert len(np.where(adjacency_matrix > 0)[0]) == 522560 * 2
print(graph.get_edge_list())
def test_sliced_wasserstein_gram_matrix(benchmark):
simple_archi: Architecture = Architecture(
preprocess=lambda x: x, layers=[LinearLayer(4, 3), SoftMaxLayer()]
)
simple_archi.build_matrices()
embeddings = list()
for val in np.random.randint(1, 100, 100):
idx = np.random.randint(0, 4)
z = np.zeros(4)
z[idx] = val
ex = torch.from_numpy(z)
g = Graph.from_architecture_and_data_point(simple_archi, ex)
dgm = compute_dgm_from_graph(g)
embeddings.append(dgm)
def compute_matrix():
return get_gram_matrix(
kernel_type=KernelType.SlicedWasserstein,
embeddings_in=embeddings,
embeddings_out=embeddings,
params=[{"M": 50, "sigma": 0.5}],
)[0]
legacy_matrix = get_gram_matrix_legacy(
kernel_type=KernelType.SlicedWasserstein,
embeddings_in=embeddings,
embeddings_out=embeddings,
params={"M": 50, "sigma": 0.5},
)
print(legacy_matrix)
matrix = benchmark(compute_matrix)
print(matrix)
assert np.isclose(np.linalg.norm(matrix - legacy_matrix), 0.0)
def get_embedding(
embedding_type: str,
line: DatasetLine,
architecture: Architecture,
quantiles_helpers_for_sigmoid=None
) -> Embedding:
time_taken = dict()
if line.graph is None:
start = time.time()
graph = Graph.from_architecture_and_data_point(
architecture=architecture, x=line.x.double()
)
time_taken["graph"] = time.time()-start
else:
graph = line.graph
if quantiles_helpers_for_sigmoid is not None:
start = time.time()
graph.sigmoidize(quantiles_helpers=quantiles_helpers_for_sigmoid)
time_taken["sigmoidize"] = time.time() - start
if embedding_type == EmbeddingType.PersistentDiagram:
start = time.time()
dgm = compute_dgm_from_graph(graph)
del graph
time_taken[f"E_{embedding_type}"] = time.time() - start
return Embedding(value=dgm, time_taken=time_taken)
elif embedding_type == EmbeddingType.RawGraph:
start = time.time()
mat = to_sparse_vector(graph.get_adjacency_matrix())
del graph
time_taken[f"E_{embedding_type}"] = time.time() - start
return Embedding(value=mat, time_taken=time_taken)
else:
raise NotImplementedError(embedding_type)
)
from tda.graph import Graph
from tda.models import Architecture
from tda.models.architectures import LinearLayer, SoftMaxLayer
simple_archi: Architecture = Architecture(
preprocess=lambda x: x,
layers=[LinearLayer(4, 3), LinearLayer(3, 2), LinearLayer(2, 10), SoftMaxLayer()],
)
ex1 = torch.ones(4) * 42
ex2 = torch.ones(4) * 37
simple_archi.build_matrices()
g1 = Graph.from_architecture_and_data_point(simple_archi, ex1)
g2 = Graph.from_architecture_and_data_point(simple_archi, ex2)
dgm1 = compute_dgm_from_graph(g1, astuple=False)
dgm2 = compute_dgm_from_graph(g2, astuple=False)
dgm1_tuple = compute_dgm_from_graph(g1, astuple=True)
dgm2_tuple = compute_dgm_from_graph(g2, astuple=True)
def test_simple_edge_list():
edge_list = [((0, 1), np.float64(3.0)), ((0, 2), np.float64(2.0))]
ret = compute_dgm_from_edges(edge_list)
def compute_dgm_from_graph(graph: Graph, astuple: bool = True, negate: bool = True):
return compute_dgm_from_edges(graph.get_edge_list(), astuple=astuple, negate=negate)
def get_sample_dataset(
epsilon: float,
noise: float,
adv: bool,
dataset: Dataset,
train: bool,
succ_adv: bool,
archi: Architecture = mnist_mlp,
dataset_size: int = 100,
attack_type: str = "FGSM",
attack_backend: str = AttackBackend.FOOLBOX,
num_iter: int = 10,
offset: int = 0,
per_class: bool = False,
compute_graph: bool = False,
transfered_attacks: bool = False,
) -> typing.List[DatasetLine]:
logger.info(f"Using source dataset {dataset.name}")
logger.info(f"Checking that the received architecture has been trained")
assert archi.is_trained
logger.info(f"OK ! Architecture is ready")
logger.info(
f"I am going to generate a dataset of {dataset_size} points...")
if adv:
logger.info(
f"Only successful adversaries ? {'yes' if succ_adv else 'no'}")
logger.info(f"Which attack ? {attack_type}")
logger.info(f"Which backend ? {attack_backend}")
else:
logger.info("This dataset will be non-adversarial !")
if transfered_attacks:
logger.info(
f"Loading the architecture to generate adversaries with transferred attacks with {archi.epochs} epochs"
)
archi = architecture = get_deep_model(
num_epochs=archi.epochs,
dataset=dataset,
architecture=archi,
train_noise=archi.train_noise,
)
if archi.epochs % 10 == 0:
archi.epochs += 1
list_locals = locals()
logger.info(f"locals = {list_locals}")
source_dataset_path = get_my_path(list_locals)
if os.path.exists(source_dataset_path):
source_dataset = torch.load(source_dataset_path)
source_dataset_size = len(source_dataset)
logger.info(
f"Successfully loaded dataset of trsf attacks (len {source_dataset_size})"
)
archi.epochs -= 1
current_sample_id = 0
else:
source_dataset = (dataset.train_dataset
if train else dataset.test_and_val_dataset)
source_dataset_size = len(source_dataset)
current_sample_id = offset
else:
source_dataset = (dataset.train_dataset
if train else dataset.test_and_val_dataset)
source_dataset_size = len(source_dataset)
current_sample_id = offset
final_dataset = list()
if dataset.name in ["tinyimagenet"]:
per_class_nb_samples = np.repeat(0, 200)
elif dataset.name in ["cifar100"]:
per_class_nb_samples = np.repeat(0, 100)
else:
per_class_nb_samples = np.repeat(0, 10)
#current_sample_id = offset
dataset_done = False
batch_size = 32
while not dataset_done and current_sample_id < source_dataset_size:
samples = None
processed_samples = None
y_pred = None
while processed_samples is None and current_sample_id < source_dataset_size:
if False: #transfered_attacks:
samples = (
source_dataset["x"][current_sample_id:current_sample_id +
batch_size],
source_dataset["y"][current_sample_id:current_sample_id +
batch_size],
)
if adv:
processed_samples = (
source_dataset["x_adv"]
[current_sample_id:current_sample_id + batch_size],
source_dataset["y"]
[current_sample_id:current_sample_id + batch_size],
)
else:
processed_samples = samples
else:
# Fetching a batch of samples and concatenating them
batch = source_dataset[current_sample_id:current_sample_id +
batch_size]
if isinstance(batch[0], DatasetLine):
x = torch.cat([torch.unsqueeze(s.x, 0) for s in batch],
0).to(device)
y = np.array([s.y for s in batch])
logger.info(f"shape of x = {x.shape}")
else:
x = torch.cat([torch.unsqueeze(s[0], 0) for s in batch],
0).to(device)
y = np.array([s[1] for s in batch])
samples = (x, y)
# Calling process_sample on the batch
# TODO: Ensure process_sample handles batched examples
processed_samples = process_sample(
sample=samples,
adversarial=adv,
noise=noise,
epsilon=epsilon,
model=archi,
attack_type=attack_type,
num_iter=num_iter,
attack_backend=attack_backend,
)
# Increasing current_sample_id
current_sample_id += batch_size
assert (samples[1] == processed_samples[1]).all()
y_pred = archi(processed_samples[0]).argmax(dim=-1).cpu().numpy()
if adv and succ_adv:
# Check where the attack was successful
valid_attacks = np.where(samples[1] != y_pred)[0]
logger.debug(
f"Attack succeeded on {len(valid_attacks)} points over {len(samples[1])}"
)
if len(valid_attacks) == 0:
processed_samples = None
else:
processed_samples = (
processed_samples[0][valid_attacks],
processed_samples[1][valid_attacks],
)
samples = (
samples[0][valid_attacks],
samples[1][valid_attacks],
)
# If the while loop did not return any samples, let's stop here
if processed_samples is None:
break
# Compute the norms on the batch
l2_norms = (torch.norm(
(processed_samples[0].double() - samples[0].double()).flatten(1),
p=2,
dim=1,
).cpu().detach().numpy())
linf_norms = (torch.norm(
(processed_samples[0].double() - samples[0].double()).flatten(1),
p=float("inf"),
dim=1,
).cpu().detach().numpy())
# Update the counter per class
for clazz in processed_samples[1]:
per_class_nb_samples[clazz] += 1
# Unbatching and return DatasetLine
# TODO: see if we can avoid unbatching
for i in range(len(processed_samples[1])):
x = torch.unsqueeze(processed_samples[0][i], 0).double()
# (OPT) Compute the graph
graph = (Graph.from_architecture_and_data_point(
architecture=archi, x=x) if compute_graph else None)
# Add the line to the dataset
final_dataset.append(
DatasetLine(
graph=graph,
x=x,
y=processed_samples[1][i],
y_pred=y_pred[i],
y_adv=adv,
l2_norm=l2_norms[i],
linf_norm=linf_norms[i],
sample_id=current_sample_id,
))
# Are we done yet ?
if not per_class:
# We are done if we have enough points in the dataset
dataset_done = len(final_dataset) >= dataset_size
else:
# We are done if all classes have enough points
dataset_done = all(
np.asarray(per_class_nb_samples) >= dataset_size)
if dataset_done:
break
logger.info(f"Compputed {len(final_dataset)}/{dataset_size} samples.")
if len(final_dataset) < dataset_size:
logger.warn(
f"I was only able to generate {len(final_dataset)} points even if {dataset_size} was requested. "
f"This is probably a lack of adversarial points.")
return final_dataset