def test_data_logger_val_data_dicts(test_settings, live_mock_server):
with wandb.init(settings=test_settings) as run:
vd = ValidationDataLogger(
inputs={
"ia": np.array([[i, i, i] for i in range(10)]),
"ib": np.array([[i, i, i] for i in range(10)]),
},
targets={
"ta": np.array([[i] for i in range(10)]),
"tb": np.array([[i] for i in range(10)]),
},
indexes=None,
validation_row_processor=None,
prediction_row_processor=None,
class_labels=None,
infer_missing_processors=False,
)
cols = ["ia", "ib", "ta", "tb"]
tcols = vd.validation_indexes[0]._table.columns
assert set(tcols) == set(cols)
assert np.all([
vd.validation_indexes[0]._table.data[i][tcols.index(
"ia")].tolist() == [i, i, i]
and vd.validation_indexes[0]._table.data[i][tcols.index(
"ib")].tolist() == [i, i, i]
and vd.validation_indexes[0]._table.data[i][tcols.index(
"ta")].tolist() == [i]
and vd.validation_indexes[0]._table.data[i][tcols.index(
"tb")].tolist() == [i] for i in range(10)
])
assert (
vd.validation_indexes[0]._table._get_artifact_reference_entry()
is not None)
def test_data_logger_pred_user_proc(test_settings, live_mock_server):
with wandb.init(settings=test_settings) as run:
vd = ValidationDataLogger(
inputs=np.array([[i, i, i] for i in range(10)]),
targets=np.array([[i] for i in range(10)]),
indexes=None,
validation_row_processor=None,
prediction_row_processor=lambda ndx, row: {"oa": row["output"] + 1},
class_labels=None,
infer_missing_processors=False,
)
t = vd.log_predictions(vd.make_predictions(lambda inputs: inputs[:, 0]))
cols = ["val_row", "output", "oa"]
tcols = t.columns
assert set(tcols) == set(cols)
assert np.all([t.data[i] == [i, i, i + 1] for i in range(10)])
assert t._get_artifact_reference_entry() is not None
def test_data_logger_val_invalid(test_settings, live_mock_server):
with wandb.init(settings=test_settings) as run:
with pytest.raises(AssertionError):
vd = ValidationDataLogger(
inputs={
"ia": np.array([[i, i, i] for i in range(10)]),
"ib": np.array([[i, i, i] for i in range(10)]),
},
targets=None,
indexes=None,
validation_row_processor=None,
prediction_row_processor=None,
class_labels=None,
infer_missing_processors=False,
)
def test_data_logger_val_indexes(test_settings, live_mock_server):
with wandb.init(settings=test_settings) as run:
table = wandb.Table(columns=["label"], data=[["cat"]])
vd = ValidationDataLogger(
inputs={
"ia": np.array([[i, i, i] for i in range(10)]),
"ib": np.array([[i, i, i] for i in range(10)]),
},
targets=None,
indexes=[table.index_ref(0) for i in range(10)],
validation_row_processor=None,
prediction_row_processor=None,
class_labels=None,
infer_missing_processors=False,
)
def test_data_logger_val_user_proc(test_settings, live_mock_server):
with wandb.init(settings=test_settings) as run:
vd = ValidationDataLogger(
inputs=np.array([[i, i, i] for i in range(10)]),
targets=np.array([[i] for i in range(10)]),
indexes=None,
validation_row_processor=lambda ndx, row: {
"ip_1": row["input"] + 1,
"tp_1": row["target"] + 1,
},
prediction_row_processor=None,
class_labels=None,
infer_missing_processors=False,
)
cols = [
"input",
"target",
"ip_1",
"tp_1",
]
tcols = vd.validation_indexes[0]._table.columns
assert set(tcols) == set(cols)
assert np.all(
[
vd.validation_indexes[0]._table.data[i][tcols.index("input")].tolist()
== [i, i, i]
and vd.validation_indexes[0]
._table.data[i][tcols.index("target")]
.tolist()
== [i]
and vd.validation_indexes[0]
._table.data[i][tcols.index("ip_1")]
.tolist()
== [i + 1, i + 1, i + 1]
and vd.validation_indexes[0]
._table.data[i][tcols.index("tp_1")]
.tolist()
== [i + 1]
for i in range(10)
]
)
assert (
vd.validation_indexes[0]._table._get_artifact_reference_entry() is not None
)
def on_train_begin(self, logs=None):
if self.log_evaluation:
try:
validation_data = None
if self.validation_data:
validation_data = self.validation_data
elif self.generator:
if not self.validation_steps:
wandb.termwarn(
"WandbCallback is unable to log validation data. When using a generator for validation_data, you must pass validation_steps"
)
else:
x = None
y_true = None
for i in range(self.validation_steps):
bx, by_true = next(self.generator)
if x is None:
x, y_true = bx, by_true
else:
x, y_true = (
np.append(x, bx, axis=0),
np.append(y_true, by_true, axis=0),
)
validation_data = (x, y_true)
else:
wandb.termwarn(
"WandbCallback is unable to read validation_data from trainer and therefore cannot log validation data. Ensure Keras is properly patched by calling `from wandb.keras import WandbCallback` at the top of your script."
)
if validation_data:
self._validation_data_logger = ValidationDataLogger(
inputs=validation_data[0],
targets=validation_data[1],
indexes=self._validation_indexes,
validation_row_processor=self.
_validation_row_processor,
prediction_row_processor=self.
_prediction_row_processor,
class_labels=self.labels,
infer_missing_processors=self.
_infer_missing_processors,
)
except Exception as e:
wandb.termwarn(
"Error initializing ValidationDataLogger in WandbCallback. Skipping logging validation data. Error: "
+ str(e))
def _init_validation_gen(self):
"""
Helper method for initializing Validation data table
"""
if self.log_evaluation:
try:
validation_data = None
if self.validation_data:
validation_data = self.validation_data
self.validation_data_logger = ValidationDataLogger(
inputs=validation_data[0],
targets=validation_data[1],
indexes=None,
validation_row_processor=None,
prediction_row_processor=lambda ndx, row: {"output": np.argmax(row["output"])},
class_labels=self.labels,
infer_missing_processors=self.infer_missing_processors)
except Exception as e:
wandb.termwarn(
"Error initializing ValidationDataLogger in WandbCallback. Skipping logging validation data. Error: " + str(
e))
def _init_testing_gen(self):
"""
Helper method for initializing Testing data table
"""
if self.log_evaluation:
try:
testing_data = None
if self.testing_data:
testing_data = self.testing_data
self.testing_data_logger = ValidationDataLogger(
inputs=testing_data[0],
targets=testing_data[1],
indexes=None,
validation_row_processor=None,
prediction_row_processor=None,
class_labels=self.labels,
infer_missing_processors=self.infer_missing_processors)
except Exception as e:
wandb.termwarn(
"Error initializing ValidationDataLogger in WandbCallback. Skipping logging validation data. Error: " + str(
e))
def test_data_logger_val_data_lists(live_mock_server, test_settings):
with wandb.init(settings=test_settings) as run:
vd = ValidationDataLogger(
inputs=np.array([[i, i, i] for i in range(10)]),
targets=np.array([[i] for i in range(10)]),
indexes=None,
validation_row_processor=None,
prediction_row_processor=None,
class_labels=None,
infer_missing_processors=False,
)
cols = ["input", "target"]
tcols = vd.validation_indexes[0]._table.columns
assert set(tcols) == set(cols)
assert np.all([
vd.validation_indexes[0]._table.data[i][tcols.index(
"input")].tolist() == [i, i, i]
and vd.validation_indexes[0]._table.data[i][tcols.index(
"target")].tolist() == [i] for i in range(10)
])
assert vd.validation_indexes[0]._table._get_artifact_entry_ref_url(
) is not None
run.finish()
def test_data_logger_pred_inferred_proc_no_classes(test_settings,
live_mock_server):
with wandb.init(settings=test_settings) as run:
vd = ValidationDataLogger(
inputs=np.array([[i, i, i] for i in range(10)]),
targets=np.array([[i] for i in range(10)]),
indexes=None,
validation_row_processor=None,
prediction_row_processor=None,
class_labels=None,
infer_missing_processors=True,
)
t = vd.log_predictions(
vd.make_predictions(
lambda inputs: {
"simple": np.random.randint(5, size=(10)),
"wrapped": np.random.randint(5, size=(10, 1)),
"logits": np.random.randint(5, size=(10, 5)),
"nodes": np.random.randint(5, size=(10, 10)),
"2dimages": np.random.randint(255, size=(10, 5, 5)),
"3dimages": np.random.randint(255, size=(10, 5, 5, 3)),
"video": np.random.randint(255, size=(10, 5, 5, 3, 10)),
}))
cols = [
"val_row",
"simple",
"wrapped",
"logits",
"nodes",
"2dimages",
"3dimages",
"video",
"wrapped:val",
"logits:node",
"logits:argmax",
"logits:argmin",
"nodes:node",
"nodes:argmax",
"nodes:argmin",
]
if CAN_INFER_IMAGE_AND_VIDEO:
cols.append("2dimages:image")
cols.append("3dimages:image")
cols.append("video:video")
tcols = t.columns
row = t.data[0]
assert set(tcols) == set(cols)
assert isinstance(row[tcols.index("val_row")],
wandb.data_types._TableIndex)
assert isinstance(row[tcols.index("simple")].tolist(), int)
assert len(row[tcols.index("wrapped")]) == 1
assert len(row[tcols.index("logits")]) == 5
assert len(row[tcols.index("nodes")]) == 10
assert row[tcols.index("2dimages")].shape == (5, 5)
assert row[tcols.index("3dimages")].shape == (5, 5, 3)
assert row[tcols.index("video")].shape == (5, 5, 3, 10)
assert isinstance(row[tcols.index("wrapped:val")].tolist(), int)
# assert isinstance(row[tcols.index("logits:node")], dict)
assert isinstance(row[tcols.index("logits:node")], list)
assert isinstance(row[tcols.index("logits:argmax")].tolist(), int)
assert isinstance(row[tcols.index("logits:argmin")].tolist(), int)
# assert isinstance(row[tcols.index("nodes:node")], dict)
assert isinstance(row[tcols.index("nodes:node")], list)
assert isinstance(row[tcols.index("nodes:argmax")].tolist(), int)
assert isinstance(row[tcols.index("nodes:argmin")].tolist(), int)
if CAN_INFER_IMAGE_AND_VIDEO:
assert isinstance(row[tcols.index("2dimages:image")],
wandb.data_types.Image)
assert isinstance(row[tcols.index("3dimages:image")],
wandb.data_types.Image)
assert isinstance(row[tcols.index("video:video")],
wandb.data_types.Video)
def test_data_logger_val_inferred_proc(test_settings, live_mock_server):
with wandb.init(settings=test_settings) as run:
np.random.seed(42)
vd = ValidationDataLogger(
inputs=np.array([[i, i, i] for i in range(10)]),
targets={
"simple": np.random.randint(5, size=(10)),
"wrapped": np.random.randint(5, size=(10, 1)),
"logits": np.random.randint(5, size=(10, 5)),
"nodes": np.random.randint(5, size=(10, 10)),
"2dimages": np.random.randint(255, size=(10, 5, 5)),
"3dimages": np.random.randint(255, size=(10, 5, 5, 3)),
"video": np.random.randint(255, size=(10, 5, 5, 3, 10)),
},
indexes=None,
validation_row_processor=None,
prediction_row_processor=None,
class_labels=["a", "b", "c", "d", "e"],
infer_missing_processors=True,
)
cols = [
"input",
"simple",
"wrapped",
"logits",
"nodes",
"2dimages",
"3dimages",
"video",
"input:node",
"input:argmax",
"input:argmin",
"wrapped:class",
"logits:max_class",
"logits:score",
"nodes:node",
"nodes:argmax",
"nodes:argmin",
]
if CAN_INFER_IMAGE_AND_VIDEO:
cols.append("2dimages:image")
cols.append("3dimages:image")
cols.append("video:video")
tcols = vd.validation_indexes[0]._table.columns
row = vd.validation_indexes[0]._table.data[0]
assert set(tcols) == set(cols)
assert np.all(row[tcols.index("input")] == [0, 0, 0])
assert isinstance(row[tcols.index("simple")].tolist(), int)
assert len(row[tcols.index("wrapped")]) == 1
assert len(row[tcols.index("logits")]) == 5
assert len(row[tcols.index("nodes")]) == 10
assert row[tcols.index("2dimages")].shape == (5, 5)
assert row[tcols.index("3dimages")].shape == (5, 5, 3)
assert row[tcols.index("video")].shape == (5, 5, 3, 10)
# assert isinstance(row[tcols.index("input:node")], dict)
assert isinstance(row[tcols.index("input:node")], list)
assert isinstance(row[tcols.index("input:argmax")].tolist(), int)
assert isinstance(row[tcols.index("input:argmin")].tolist(), int)
assert isinstance(row[tcols.index("wrapped:class")],
wandb.data_types._TableIndex)
assert isinstance(row[tcols.index("logits:max_class")],
wandb.data_types._TableIndex)
assert isinstance(row[tcols.index("logits:score")], dict)
# assert isinstance(row[tcols.index("nodes:node")], dict)
assert isinstance(row[tcols.index("nodes:node")], list)
assert isinstance(row[tcols.index("nodes:argmax")].tolist(), int)
assert isinstance(row[tcols.index("nodes:argmin")].tolist(), int)
if CAN_INFER_IMAGE_AND_VIDEO:
assert isinstance(row[tcols.index("2dimages:image")],
wandb.data_types.Image)
assert isinstance(row[tcols.index("3dimages:image")],
wandb.data_types.Image)
assert isinstance(row[tcols.index("video:video")],
wandb.data_types.Video)