def test_sequence_dset_vanilla():
D1 = DebugDataset(size=10)
D2 = DebugDataset(size=10)
D3 = DebugDataset(size=10)
D = D1 + D2 + D3
S = SequenceDataset(D, 3, fid_key="label1")
assert len(S) == (3 * (10 - 2))
assert len(S.labels["label1"]) == (3 * (10 - 2))
d = S[0]
ref = {
"val": [0, 1, 2],
"other": [0, 1, 2],
"index_": 0,
"dataset_index_": [0, 0, 0],
}
assert d == ref
l = np.array(S.labels["label1"][:3])
refl = np.array([[0, 1, 2], [1, 2, 3], [2, 3, 4]])
assert np.all(l == refl)
def test_get_seq():
config = {
"dataset": "edflow.data.believers.sequence.getSeqDataset",
"seqdataset": {
"dataset": "edflow.debug.ConfigDebugDataset",
"length": 3,
"step": 1,
"fid_key": "label1",
},
"size": 10,
}
S1 = getSeqDataset(config)
D = DebugDataset(size=10)
S2 = SequenceDataset(D, 3, fid_key="label1")
assert len(S1) == len(S2)
assert len(S1.labels["label1"]) == len(S2.labels["label1"])
assert len(S1) == len(S1.labels["label1"])
s1 = S1[0]
s2 = S2[0]
assert s1 == s2
assert np.all(S1.labels["label1"] == S2.labels["label1"])
def test_unseq_vanilla():
D1 = DebugDataset(size=10)
D2 = DebugDataset(size=10)
D = D1 + D2
S = SequenceDataset(D, 3, fid_key="label1")
U = UnSequenceDataset(S)
print(np.shape(U.labels["label1"]))
assert len(U) == len(S) * 3
assert len(U.labels["label1"]) == len(S.labels["label1"]) * 3
du = U[[0, 1, 2]]
ds = S[0]
for k in ds.keys():
if k == "index_":
continue
for i in range(3):
print(k, i)
val_u = du[i][k]
val_s = ds[k][i]
assert val_s == val_u
def test_seq_offset_fid_strat_reset():
D1 = DebugDataset(size=10)
D2 = DebugDataset(size=12, offset=3)
D = D1 + D2
S = SequenceDataset(D, 3, fid_key="label1", strategy="reset")
assert len(S) == (len(D1) - 2 + len(D2) - 2)
def test_seq_wrong_fid_len():
D1 = DebugDataset(size=10)
D2 = DebugDataset(size=10)
D = D1 + D2
D.labels["label1"] = np.array(D.labels["label1"])[:10]
with pytest.raises(ValueError):
S = SequenceDataset(D, 3, fid_key="label1")
def test_seq_wrong_fid_dtype():
D1 = DebugDataset(size=10)
D2 = DebugDataset(size=10)
D = D1 + D2
D.labels["label1"] = np.array(D.labels["label1"], dtype=float)
with pytest.raises(TypeError):
S = SequenceDataset(D, 3, fid_key="label1")
def test_sequence_dset_offset_fid():
D1 = DebugDataset(size=10)
D2 = DebugDataset(size=10)
D3 = DebugDataset(size=10, offset=3)
D4 = DebugDataset(size=10, offset=3)
D = D1 + D2 + D3 + D4
labels = np.array(D.labels["label1"])
diffs = labels[1:] - labels[:-1]
idxs = np.where(diffs <= 0)[0] + 1
with pytest.raises(ValueError):
S = SequenceDataset(D, 3, fid_key="label1")
def __init__(self, config, mode="all"):
assert mode in ["train", "validation", "all"
], f"Should be train, validation or all, got {mode}"
self.config = config
self.sequence_length = 30
self.sc = SequenceDataset(MPII_Sequence(config),
self.sequence_length,
step=config["sequence_step_size"])
self.train = int(config["train_size"] * len(self.sc))
self.test = 1 - self.train
self.sigma = config["sigma"]
self.augmentation = config["augmentation"]
self.aug_factor = 0.5
self.resize = iaa.Resize(self.config["resize_to"])
if self.augmentation:
self.seq = iaa.Sequential(
[
# iaa.Sometimes(self.aug_factor, iaa.AdditiveGaussianNoise(scale=0.05 * 255)),
# iaa.Sometimes(self.aug_factor, iaa.SaltAndPepper(0.01, per_channel=False)),
# iaa.Sometimes(self.aug_factor, iaa.CoarseDropout(0.01, size_percent=0.5)),
iaa.Fliplr(self.aug_factor),
iaa.Flipud(self.aug_factor),
# iaa.Sometimes(self.aug_factor, iaa.GaussianBlur(sigma=(0, 3.0))),
# iaa.LinearContrast((0.75, 1.5)),
# Convert each image to grayscale and then overlay the
# result with the original with random alpha. I.e. remove
# colors with varying strengths.
# iaa.Grayscale(alpha=(0.0, 1.0)),
],
random_order=True)
if mode != "all":
# split_indices = np.arange(self.train) if mode == "train" else np.arange(self.train + 1, len(self.sc))
dset_indices = np.arange(len(self.sc))
train_indices, test_indices = sklearn.model_selection.train_test_split(
dset_indices,
train_size=float(config["train_size"]),
random_state=int(config["random_state"]))
if mode == "train":
self.data = SubDataset(self.sc, train_indices)
else:
self.data = SubDataset(self.sc, test_indices)
else:
self.data = self.sc
# for k, v in d.items()]))
from edflow.debug import DebugDataset
D = DebugDataset()
def labels(data, i):
return {"fid": i}
D = ExtraLabelsDataset(D, labels)
print("D")
for k, v in D.labels.items():
print(k)
print(np.shape(v))
S = SequenceDataset(D, 2)
print("S")
for k, v in S.labels.items():
print(k)
print(np.shape(v))
S = SubDataset(S, [2, 5, 10])
print("Sub")
for k, v in S.labels.items():
print(k)
print(np.shape(v))
U = UnSequenceDataset(S)
print("U")
for k, v in U.labels.items():
print(k)
def __init__(self, config, mode="all"):
assert mode in ["train", "validation", "all"], f"Should be train, validation or all, got {mode}"
self.config = config
self.sequence_length = 2 # if config.get("sequence_length", False) == False else config["sequence_length"]
self.sc = SequenceDataset(Animal_Sequence(config), self.sequence_length, step=config["sequence_step_size"])
# works if dataroot like "VOC2011/cats_meta"
# TODO PROBABLY NOT CORRECT HERE
self.animal = config["dataroot"].split("/")[1].split("_")[0]
self.train = int(config["train_size"] * len(self.sc))
self.test = 1 - self.train
self.sigma = config["sigma"]
self.augmentation = config["augmentation"]
self.aug_factor = 0.5
self.logger = get_logger(self)
self.resize = iaa.Resize(self.config["resize_to"])
if self.augmentation:
self.seq = iaa.Sequential([
iaa.Sometimes(self.aug_factor, iaa.AdditiveGaussianNoise(scale=0.05 * 255)),
iaa.Sometimes(self.aug_factor, iaa.SaltAndPepper(0.01, per_channel=False)),
iaa.Sometimes(self.aug_factor, iaa.CoarseDropout(0.01, size_percent=0.5)),
iaa.Fliplr(self.aug_factor),
iaa.Flipud(self.aug_factor),
iaa.Sometimes(self.aug_factor, iaa.GaussianBlur(sigma=(0, 3.0))),
iaa.LinearContrast((0.75, 1.5)),
# Convert each image to grayscale and then overlay the
# result with the original with random alpha. I.e. remove
# colors with varying strengths.
iaa.Grayscale(alpha=(0.0, 1.0)),
], random_order=True)
self.joints = [
[2, 0], # Nose - L_Eye
[2, 1], # Nose - R_Eye
[0, 3], # L_Eye - L_EarBase
[1, 4], # R_Eye - R_EarBase
[2, 8], # Nose - Throat
[8, 9], # Throat - L_F_Elbow
[8, 5], # Throat - R_F_Elbow
[9, 16], # L_F_Elbow - L_F_Knee
[16, 6], # L_F_Knee - L_F_Paw
[5, 17], # R_F_Elbow - R_F_Knee
[17, 7], # R_F_Knee - R_F_Paw
[14, 18], # L_B_Elbow - L_B_Knee
[18, 13], # L_B_Knee - L_B_Paw
[15, 19], # R_B_Elbow - R_B_Knee
[19, 13], # R_B_Knee - R_B_Paw
[10, 11], # Withers - TailBase
]
if mode != "all":
# split_indices = np.arange(self.train) if mode == "train" else np.arange(self.train + 1, len(self.sc))
dset_indices = np.arange(len(self.sc))
train_indices, test_indices = sklearn.model_selection.train_test_split(dset_indices,
train_size=float(
config["train_size"]),
random_state=int(
config["random_state"]))
if mode == "train":
self.data = SubDataset(self.sc, train_indices)
else:
self.data = SubDataset(self.sc, test_indices)
else:
self.data = self.sc
def test_seq_offset_fid_strat_unknown():
D = DebugDataset(size=12, offset=3)
with pytest.raises(ValueError):
S = SequenceDataset(D, 3, fid_key="label1", strategy="recolorize")