def test_complex_mean(self, concept: Concept):
"""Test obtaining the mean concept embedding, meaning the optimal
mean hyperplane."""
concept_args = dict(concept=concept, model_stump=None)
for desc, (embs, (m_w, m_b)) in self.INTUITIVE_MEAN_EXAMPLES.items():
m_w: np.ndarray = np.array(m_w)
embeddings = [
ConceptEmbedding(normal_vec=w,
support_factor=b,
**concept_args) for w, b in embs
]
# Actual routine
m_emb: ConceptEmbedding = ConceptEmbedding.mean_by_angle(
embeddings)
context_info = (("context:\n mean embedding: ({}, {})"
"\n in embeddings ({}) as (normal vec, support, "
"scaling):\n {}").format(
m_emb.normal_vec, m_emb.support_factor, desc,
[(e.normal_vec, e.support_factor,
e.scaling_factor) for e in embeddings]))
# Format checks
assert m_emb.normal_vec.shape == embeddings[0].normal_vec.shape
assert np.array(m_emb.support_factor).shape == np.array(
embeddings[0].support_factor).shape
# Value checks
assert np.allclose(m_emb.normal_vec, m_w), \
("Wrong mean normal weight: expected {}, but was {}; {}"
.format(m_w, m_emb.normal_vec, context_info))
assert np.allclose(m_emb.support_factor, m_b), \
("Wrong mean normal bias: expected {}, but was {}; {}"
.format(m_b, m_emb.support_factor, context_info))
def test_distance_mean_condition(self, concept: Concept):
"""Test that the mean embedding by distance really fulfills the
distance condition.
This is that at any point in space the distance of the mean embedding is
the mean of the distances of the other embeddings."""
# test points for testing the distance condition in several dimensions
concept_args = dict(concept=concept, model_stump=None)
for desc, (embs, _) in self.DISTANCE_MEAN_EXAMPLES.items():
embeddings = [
ConceptEmbedding(normal_vec=w,
support_factor=b,
**concept_args) for w, b in embs
]
# Actual routine
m_emb: ConceptEmbedding = \
ConceptEmbedding.mean_by_distance(embeddings)
dims: int = m_emb.normal_vec.size
test_points = self.TEST_POINTS_nD[dims]
for point in test_points:
dist_mean: float = m_emb.distance(point)
dists: List[float] = [e.distance(point) for e in embeddings]
mean_dist: float = float(np.mean(dists))
assert np.allclose(dist_mean, mean_dist), \
(("In point {}, distance of mean ({}) is not mean of "
"distances {} ({}) for\n in embeddings: {}:\n {}"
"\n mean embedding: {}")
.format(point, dist_mean, dists, mean_dist,
desc, [(e.normal_vec, e.support_factor)
for e in embeddings],
(m_emb.normal_vec, m_emb.support_factor)))
def test_complex_mean_condition(self, concept: Concept):
"""Test obtaining the mean concept embedding, meaning the optimal mean
hyperplane.
The condition for the more complex mean here is that at any point the
mean distances to the hyperplanes along the normal vector equal the
(unscaled!) distance to the mean hyperplane."""
concept_args = dict(concept=concept, model_stump=None)
for desc, (embs, _) in self.INTUITIVE_MEAN_EXAMPLES.items():
embs: List[ConceptEmbedding] = \
[ConceptEmbedding(normal_vec=w, support_factor=b,
**concept_args) for w, b in embs]
# Actual routine
m_emb: ConceptEmbedding = ConceptEmbedding.mean_by_angle(embs) \
.normalize().to_pos_scaling()
context_info = ((
"embeddings in context as (normal vec, support, scaling):"
"\n mean embedding: ({}, {}, {})"
"\n in embeddings ({}):\n {}").format(
m_emb.normal_vec, m_emb.support_factor,
m_emb.scaling_factor, desc,
[(e.normal_vec, e.support_factor, e.scaling_factor)
for e in embs]))
for point in self.TEST_POINTS_nD[m_emb.normal_vec.size]:
# The normalized vectors with no scaling for distance
# calculation
# pylint: disable=no-member
normed_embs: List[ConceptEmbedding] = \
[e.normalize().to_pos_scaling().forget_scaling()
for e in embs]
# pylint: enable=no-member
# The for each hyperplane the cosine of its angle to the mean
# hyperplane
cos_angles = [
cos_angle(e.normal_vec, m_emb.normal_vec)
for e in normed_embs
]
# Distance from each hyperplane to the point along the
# hyperplane's normal vec
plane_to_pt_dists: List[float] = [
e.distance(point) for e in normed_embs
]
# Distances from the point to each hyperplane along the mean
# normal vec
dists: List[float] = [
d / cos_a
for d, cos_a in zip(plane_to_pt_dists, cos_angles)
]
# Condition: The distances along the mean normal vec should
# sum up to the mean dist
dist_to_mean = m_emb.forget_scaling().distance(point)
assert np.allclose(dist_to_mean, np.mean(dists)), \
(("In point {}, distance to mean hyperplane is not the "
"mean distance to the other hyperplanes along the mean "
"hyperplane normal vector!"
"\ndists to hyperplanes along mean normal vec: {}, "
"mean: {};\nunscaled dist to mean hyperplane: {}; {}")
.format(point, dists, np.mean(dists), dist_to_mean,
context_info))
def test_scale(self):
"""Test fundamental properties of applying the scaling factor."""
# normal_vec, support_factor, scaling_factor
orig = ([3, 4], 1, 1)
normal_vec, support_factor, scaling_factor = orig
# noinspection PyTypeChecker
emb = ConceptEmbedding(normal_vec=normal_vec,
support_factor=support_factor,
scaling_factor=scaling_factor)
# Scaling yields new instance and old one is not changed:
assert emb is not emb.scale(), "Scaling did not yield new instance"
assert np.allclose(emb.normal_vec, np.array(normal_vec))
assert np.allclose(emb.support_factor, np.array(support_factor))
assert np.allclose(emb.scaling_factor, np.array(scaling_factor))
# Scaling embedding of scaling factor 1 does nothing
scaled_unchanged_emb = emb.scale()
assert np.allclose(emb.normal_vec, scaled_unchanged_emb.normal_vec)
assert np.allclose(emb.support_factor,
scaled_unchanged_emb.support_factor)
assert np.allclose(emb.scaling_factor,
scaled_unchanged_emb.scaling_factor)
# Normalization and then scaling should yield the same embedding:
backscaled_emb = emb.normalize().scale()
for key, (expected, obtained) in \
{"normal_vec": (emb.normal_vec, backscaled_emb.normal_vec),
"support_factor": (emb.support_factor,
backscaled_emb.support_factor),
"scaling_factor": (emb.scaling_factor,
backscaled_emb.scaling_factor)
}.items():
assert np.allclose(obtained, expected), \
("Wrong normalized {}: expected {}, but was {}"
.format(key, expected, obtained))
# Simple scaling example: scale by 2
emb.scaling_factor = 2
new = ([6, 8], 0.5, 1)
normal_vec, support_factor, scaling_factor = new
scaled_emb = emb.scale()
assert np.allclose(normal_vec, scaled_emb.normal_vec)
assert np.allclose(support_factor, scaled_emb.support_factor)
assert np.allclose(scaling_factor, scaled_emb.scaling_factor)
# Another scaling example: scale by -2
emb.scaling_factor = -2
new = ([-6, -8], -0.5, 1)
normal_vec, support_factor, scaling_factor = new
scaled_emb = emb.scale()
assert np.allclose(normal_vec, scaled_emb.normal_vec)
assert np.allclose(support_factor, scaled_emb.support_factor)
assert np.allclose(scaling_factor, scaled_emb.scaling_factor)
def test_distance_mean_exceptions(self, concept: SegmentationConcept2D):
"""Test obtaining the mean concept embedding, meaning the optimal mean
hyperplane. Are appropriate errors raised?"""
with pytest.raises(ValueError):
# Two parallels equally distributed around zero but with opposite
# pos/neg sides
concept_args = dict(concept=concept, model_stump=None)
embeddings = [
ConceptEmbedding(normal_vec=w,
support_factor=b,
**concept_args)
for w, b in [([0, 1], 2), ([0, -1], 2)]
]
ConceptEmbedding.mean_by_distance(embeddings)
with pytest.raises(ValueError):
ConceptEmbedding.mean_by_distance([])
def test_mean_results(self, concept: Concept):
"""Test obtaining the mean concept embedding by sample values."""
concept_args = dict(concept=concept, model_stump=None)
for desc, (embs, (m_w, m_b)) in self.INTUITIVE_MEAN_EXAMPLES.items():
m_w: np.ndarray = np.array(m_w)
embeddings = [
ConceptEmbedding(normal_vec=w,
support_factor=b,
**concept_args) for w, b in embs
]
# Actual routine
m_emb: ConceptEmbedding = ConceptEmbedding.mean(embeddings)
context_info = (("context:\n mean embedding: ({}, {}, 1.)"
"\n in embeddings ({}) as (normal vec, support, "
"scaling):\n {}").format(
m_emb.normal_vec, m_emb.support_factor, desc,
[(e.normal_vec, e.support_factor,
e.scaling_factor) for e in embeddings]))
# Format checks
assert m_emb.normal_vec.shape == embeddings[0].normal_vec.shape
assert np.array(m_emb.support_factor).shape == np.array(
embeddings[0].support_factor).shape
# Value checks
# Scaling
expected_scaling: float = float(
np.mean([
e.scaling_factor
for e in [e.normalize() for e in embeddings]
]))
assert float(m_emb.scaling_factor) == expected_scaling, \
("Mean scaling wrong: expected {}., but was {}; {}"
.format(expected_scaling, m_emb.scaling_factor, context_info))
# Normal vector
assert np.allclose(m_emb.normal_vec, m_w), \
("Wrong mean normal vector: expected {}, but was {}; {}"
.format(m_w, m_emb.normal_vec, context_info))
# Support
assert np.allclose(m_emb.support_factor, m_b), \
("Wrong mean support factor: expected {}, but was {}; {}"
.format(m_b, m_emb.support_factor, context_info))
def test_distance_mean_results(self, concept: Concept):
"""Test obtaining the mean concept embedding by sample values."""
concept_args = dict(concept=concept, model_stump=None)
for desc, (embs, (m_w, m_b)) in self.DISTANCE_MEAN_EXAMPLES.items():
m_w: np.ndarray = np.array(m_w)
embeddings = [
ConceptEmbedding(normal_vec=w,
support_factor=b,
**concept_args) for w, b in embs
]
# Actual routine
m_emb: ConceptEmbedding = \
ConceptEmbedding.mean_by_distance(embeddings)
context_info = (("context:\n mean embedding: ({}, {}, 1.)"
"\n in embeddings ({}) as (normal vec, support, "
"scaling):\n {}").format(
m_emb.normal_vec, m_emb.support_factor, desc,
[(e.normal_vec, e.support_factor,
e.scaling_factor) for e in embeddings]))
# Format checks
assert m_emb.normal_vec.shape == embeddings[0].normal_vec.shape
assert np.array(m_emb.support_factor).shape == np.array(
embeddings[0].support_factor).shape
# Value checks
# the embedding should be scaled
assert float(m_emb.scaling_factor) == 1., \
("Mean embedding not scaled: expected 1., but was {}; {}"
.format(m_emb.scaling_factor, context_info))
assert np.allclose(m_emb.normal_vec, m_w), \
("Wrong mean normal vector: expected {}, but was {}; {}"
.format(m_w, m_emb.normal_vec, context_info))
# For all given ground truths of support factors, check them:
if m_b is not None:
assert np.allclose(m_emb.support_factor, m_b), \
("Wrong mean support factor: expected {}, but was {}; {}"
.format(m_b, m_emb.support_factor, context_info))
def test_upper_sphere_neg_bias(self):
"""Test embedding normalization with negative bias."""
weight = np.array([-3, 4]) # |weight| = 5
bias = -1
normed_weight = np.array([3 / 5, -4 / 5])
normed_bias = 5
# noinspection PyTypeChecker
emb = ConceptEmbedding(normal_vec=weight, support_factor=bias)
normed_emb = emb.unique_upper_sphere()
# Format checks
assert normed_emb.normal_vec.shape == emb.normal_vec.shape
assert np.array(normed_emb.support_factor).shape == np.array(
emb.support_factor).shape
# Value checks
assert np.allclose(normed_emb.normal_vec, normed_weight), \
("Wrong normalized weight: expected {}, but was {}"
.format(normed_weight, normed_emb.normal_vec))
assert np.allclose(normed_emb.support_factor, normed_bias), \
("Wrong normalized bias: expected {}, but was {}"
.format(normed_bias, normed_emb.support_factor))
def to_emb(emb_vals: Sequence[float],
concept: Concept = None) -> ConceptEmbedding:
"""Given a tuple of (normal_vec, support_factor, scaling_factor) yield
embedding.
``model_stump`` is set to None."""
assert 2 <= len(emb_vals) <= 3
concept_args = dict(concept=concept,
model_stump=None,
normal_vec=emb_vals[0],
support_factor=emb_vals[1])
if len(emb_vals) == 3:
concept_args["scaling_factor"] = emb_vals[2]
return ConceptEmbedding(**concept_args)
def test_normalize(self):
"""Test fundamental properties of normalization."""
normal_vec = np.array([3, 4]) # |normal_vec| = 5
support_factor = 1
scaling_factor = 1.
normed_normal_vec = np.array([3 / 5, 4 / 5])
normed_support_factor = 5
normed_scaling_factor = 5
# noinspection PyTypeChecker
emb = ConceptEmbedding(normal_vec=normal_vec,
support_factor=support_factor)
assert emb.scaling_factor == scaling_factor, \
"Scaling factor wrongly initialized."
normed_emb = emb.normalize()
# Normalization yields new instance
assert normed_emb is not emb, "Normalization did not yield new instance"
# Format checks
assert normed_emb.normal_vec.shape == emb.normal_vec.shape
assert np.array(normed_emb.support_factor).shape == np.array(
emb.support_factor).shape
assert np.array(normed_emb.scaling_factor).shape == np.array(
emb.scaling_factor).shape
# Value checks
for key, (expected, obtained) in \
{"normal_vec": (normed_normal_vec, normed_emb.normal_vec),
"support_factor": (normed_support_factor,
normed_emb.support_factor),
"scaling_factor": (normed_scaling_factor,
normed_emb.scaling_factor)
}.items():
assert np.allclose(obtained, expected), \
("Wrong normalized {}: expected {}, but was {}"
.format(key, expected, obtained))
if not os.path.isdir(os.path.join(args.exp_root, concept)):
raise NotADirectoryError(
"Concept {} in exp_root {} is not a directory".format(
concept, args.exp_root))
# endregion
print("Starting sample generation with settings:")
key_len = max([len(k) for k in vars(args)])
print("\n".join([("{:" + str(key_len) + "}\t{}").format(k, v)
for k, v in vars(args).items()]),
flush=True)
main_model: torch.nn.Module = model_loaders.MODELS_INFO[args.model].loader(
torch.load(pkl_file, map_location=args.device))
best_embeddings: Dict[str, ConceptEmbedding] = {
concept:
ConceptEmbedding.load(os.path.join(args.exp_root, concept, 'best.npz'))
for concept in args.concepts
}
best_c_models: Dict[str, ConceptModel] = {
concept: ConceptModel.from_embedding(emb, main_model=main_model)
for concept, emb in best_embeddings.items()
}
generate(named_c_models=best_c_models,
sample_root=args.samples_root,
dest_root=args.dest_root,
recursive=args.recursive,
overwrite=args.force,
device=args.device)
def to_emb(normal_vec, support_factor, scaling_factor):
"""Shorthand to concept embedding creation"""
return ConceptEmbedding(normal_vec=normal_vec,
support_factor=support_factor,
scaling_factor=scaling_factor,
**args)
def test_complex_mean_exceptions(self):
"""Test obtaining the mean concept embedding, meaning the optimal
mean hyperplane. Are appropriate errors raised?"""
with pytest.raises(ValueError):
ConceptEmbedding.mean_by_angle([])