def log_training_progress(self, predn, path, names):
"""
Build evaluation Table. Uses reference from validation dataset table.
arguments:
predn (list): list of predictions in the native space in the format - [xmin, ymin, xmax, ymax, confidence, class]
path (str): local path of the current evaluation image
names (dict(int, str)): hash map that maps class ids to labels
"""
class_set = wandb.Classes([{'id': id, 'name': name} for id, name in names.items()])
box_data = []
total_conf = 0
for *xyxy, conf, cls in predn.tolist():
if conf >= 0.25:
box_data.append(
{"position": {"minX": xyxy[0], "minY": xyxy[1], "maxX": xyxy[2], "maxY": xyxy[3]},
"class_id": int(cls),
"box_caption": "%s %.3f" % (names[cls], conf),
"scores": {"class_score": conf},
"domain": "pixel"})
total_conf = total_conf + conf
boxes = {"predictions": {"box_data": box_data, "class_labels": names}} # inference-space
id = self.val_table_path_map[Path(path).name]
self.result_table.add_data(self.current_epoch,
id,
self.val_table.data[id][1],
wandb.Image(self.val_table.data[id][1], boxes=boxes, classes=class_set),
total_conf / max(1, len(box_data))
)
def create_dataset_table(self, dataset, class_to_id, name='dataset'):
"""
Create and return W&B artifact containing W&B Table of the dataset.
arguments:
dataset (LoadImagesAndLabels) -- instance of LoadImagesAndLabels class used to iterate over the data to build Table
class_to_id (dict(int, str)) -- hash map that maps class ids to labels
name (str) -- name of the artifact
returns:
dataset artifact to be logged or used
"""
# TODO: Explore multiprocessing to slpit this loop parallely| This is essential for speeding up the the logging
artifact = wandb.Artifact(name=name, type="dataset")
img_files = tqdm([dataset.path]) if isinstance(
dataset.path, str) and Path(dataset.path).is_dir() else None
img_files = tqdm(dataset.img_files) if not img_files else img_files
for img_file in img_files:
if Path(img_file).is_dir():
artifact.add_dir(img_file, name='data/images')
labels_path = 'labels'.join(dataset.path.rsplit('images', 1))
artifact.add_dir(labels_path, name='data/labels')
else:
artifact.add_file(img_file,
name='data/images/' + Path(img_file).name)
label_file = Path(img2label_paths([img_file])[0])
artifact.add_file(
str(label_file), name='data/labels/' +
label_file.name) if label_file.exists() else None
table = wandb.Table(columns=["id", "train_image", "Classes", "name"])
class_set = wandb.Classes([{
'id': id,
'name': name
} for id, name in class_to_id.items()])
for si, (img, labels, paths, shapes) in enumerate(tqdm(dataset)):
box_data, img_classes = [], {}
for cls, *xywh in labels[:, 1:].tolist():
cls = int(cls)
box_data.append({
"position": {
"middle": [xywh[0], xywh[1]],
"width": xywh[2],
"height": xywh[3]
},
"class_id": cls,
"box_caption": "%s" % (class_to_id[cls])
})
img_classes[cls] = class_to_id[cls]
boxes = {
"ground_truth": {
"box_data": box_data,
"class_labels": class_to_id
}
} # inference-space
table.add_data(si,
wandb.Image(paths, classes=class_set, boxes=boxes),
list(img_classes.values()),
Path(paths).name)
artifact.add(table, name)
return artifact
def test_add_obj_wbimage_classes_obj(runner):
test_folder = os.path.dirname(os.path.realpath(__file__))
im_path = os.path.join(test_folder, "..", "assets", "2x2.png")
with runner.isolated_filesystem():
artifact = wandb.Artifact(type="dataset", name="my-arty")
classes = wandb.Classes([{"id": 0, "name": "person"}])
wb_image = wandb.Image(im_path, classes=classes)
artifact.add(wb_image, "my-image")
manifest = artifact.manifest.to_manifest_json()
assert manifest["contents"] == {
"media/classes/65347c6442e21b09b198d62e080e46ce_cls.classes.json":
{
"digest": "eG00DqdCcCBqphilriLNfw==",
"size": 64,
},
"media/images/641e917f31888a48f546/2x2.png": {
"digest": "L1pBeGPxG+6XVRQk4WuvdQ==",
"size": 71,
},
"my-image.image-file.json": {
"digest": "IcEgVbPW7fE1a+g577K+VQ==",
"size": 346,
},
}
def create_dataset_table(self, dataset, class_to_id, name='dataset'):
# TODO: Explore multiprocessing to slpit this loop parallely| This is essential for speeding up the the logging
artifact = wandb.Artifact(name=name, type="dataset")
for img_file in tqdm([dataset.path]) if Path(dataset.path).is_dir() else tqdm(dataset.img_files):
if Path(img_file).is_dir():
artifact.add_dir(img_file, name='data/images')
labels_path = 'labels'.join(dataset.path.rsplit('images', 1))
artifact.add_dir(labels_path, name='data/labels')
else:
artifact.add_file(img_file, name='data/images/' + Path(img_file).name)
label_file = Path(img2label_paths([img_file])[0])
artifact.add_file(str(label_file),
name='data/labels/' + label_file.name) if label_file.exists() else None
table = wandb.Table(columns=["id", "train_image", "Classes", "name"])
class_set = wandb.Classes([{'id': id, 'name': name} for id, name in class_to_id.items()])
for si, (img, labels, paths, shapes) in enumerate(tqdm(dataset)):
height, width = shapes[0]
labels[:, 2:] = (xywh2xyxy(labels[:, 2:].view(-1, 4))) * torch.Tensor([width, height, width, height])
box_data, img_classes = [], {}
for cls, *xyxy in labels[:, 1:].tolist():
cls = int(cls)
box_data.append({"position": {"minX": xyxy[0], "minY": xyxy[1], "maxX": xyxy[2], "maxY": xyxy[3]},
"class_id": cls,
"box_caption": "%s" % (class_to_id[cls]),
"scores": {"acc": 1},
"domain": "pixel"})
img_classes[cls] = class_to_id[cls]
boxes = {"ground_truth": {"box_data": box_data, "class_labels": class_to_id}} # inference-space
table.add_data(si, wandb.Image(paths, classes=class_set, boxes=boxes), json.dumps(img_classes),
Path(paths).name)
artifact.add(table, name)
return artifact
def test_add_obj_wbimage_classes_obj_already_added(runner):
test_folder = os.path.dirname(os.path.realpath(__file__))
im_path = os.path.join(test_folder, "..", "assets", "2x2.png")
with runner.isolated_filesystem():
artifact = wandb.Artifact(type="dataset", name="my-arty")
classes = wandb.Classes([{"id": 0, "name": "person"}])
artifact.add(classes, "my-classes")
wb_image = wandb.Image(im_path, classes=classes)
artifact.add(wb_image, "my-image")
manifest = artifact.manifest.to_manifest_json()
assert manifest["contents"] == {
"my-classes.classes.json": {
"digest": "eG00DqdCcCBqphilriLNfw==",
"size": 64,
},
"media/images/641e917f/2x2.png": {
"digest": "L1pBeGPxG+6XVRQk4WuvdQ==",
"size": 71,
},
"my-image.image-file.json": {
"digest": "ksQ+BJCt+KZSsyC03K2+Uw==",
"size": 216,
},
}
def log_dataset_artifact(self, dataset, class_to_id, name='dataset'):
artifact = wandb.Artifact(name=name, type="dataset")
image_path = dataset.path
artifact.add_dir(image_path, name='data/images')
table = wandb.Table(columns=["id", "train_image", "Classes"])
class_set = wandb.Classes([{'id': id, 'name': name} for id, name in class_to_id.items()])
for si, (img, labels, paths, shapes) in enumerate(dataset):
height, width = shapes[0]
labels[:, 2:] = (xywh2xyxy(labels[:, 2:].view(-1, 4)))
labels[:, 2:] *= torch.Tensor([width, height, width, height])
box_data = []
img_classes = {}
for cls, *xyxy in labels[:, 1:].tolist():
cls = int(cls)
box_data.append({"position": {"minX": xyxy[0], "minY": xyxy[1], "maxX": xyxy[2], "maxY": xyxy[3]},
"class_id": cls,
"box_caption": "%s" % (class_to_id[cls]),
"scores": {"acc": 1},
"domain": "pixel"})
img_classes[cls] = class_to_id[cls]
boxes = {"ground_truth": {"box_data": box_data, "class_labels": class_to_id}} # inference-space
table.add_data(si, wandb.Image(paths, classes=class_set, boxes=boxes), json.dumps(img_classes))
artifact.add(table, name)
labels_path = 'labels'.join(image_path.rsplit('images', 1))
zip_path = Path(labels_path).parent / (name + '_labels.zip')
if not zip_path.is_file(): # make_archive won't check if file exists
shutil.make_archive(zip_path.with_suffix(''), 'zip', labels_path)
artifact.add_file(str(zip_path), name='data/labels.zip')
wandb.log_artifact(artifact)
print("Saving data to W&B...")
def test_add_obj_wbimage_classes_obj(runner):
test_folder = os.path.dirname(os.path.realpath(__file__))
im_path = os.path.join(test_folder, "..", "assets", "2x2.png")
with runner.isolated_filesystem():
artifact = wandb.Artifact(type="dataset", name="my-arty")
classes = wandb.Classes([{"id": 0, "name": "person"}])
wb_image = wandb.Image(im_path, classes=classes)
artifact.add(wb_image, "my-image")
manifest = artifact.manifest.to_manifest_json()
if os.name == "nt": # windows
assert manifest["contents"] == {
"classes.json": {"digest": "eG00DqdCcCBqphilriLNfw==", "size": 64},
"media\\images\\2x2.png": {
"digest": "L1pBeGPxG+6XVRQk4WuvdQ==",
"size": 71,
},
"my-image.image-file.json": {
"digest": "nD/QMrasZLE2Cp35MmshSg==",
"size": 198,
},
}
else:
assert manifest["contents"] == {
"classes.json": {"digest": "eG00DqdCcCBqphilriLNfw==", "size": 64},
"media/images/2x2.png": {
"digest": "L1pBeGPxG+6XVRQk4WuvdQ==",
"size": 71,
},
"my-image.image-file.json": {
"digest": "UhZfZLPavGE2tBRdTvIl3Q==",
"size": 196,
},
}
def log_training_progress(self, predn, path, names):
"""
Build evaluation Table. Uses reference from validation dataset table.
arguments:
predn (list): list of predictions in the native space in the format - [xmin, ymin, xmax, ymax, confidence, class]
path (str): local path of the current evaluation image
names (dict(int, str)): hash map that maps class ids to labels
"""
class_set = wandb.Classes([{
'id': id,
'name': name
} for id, name in names.items()])
box_data = []
avg_conf_per_class = [0] * len(self.data_dict['names'])
pred_class_count = {}
for *xyxy, conf, cls in predn.tolist():
if conf >= 0.25:
cls = int(cls)
box_data.append({
"position": {
"minX": xyxy[0],
"minY": xyxy[1],
"maxX": xyxy[2],
"maxY": xyxy[3]
},
"class_id": cls,
"box_caption": f"{names[cls]} {conf:.3f}",
"scores": {
"class_score": conf
},
"domain": "pixel"
})
avg_conf_per_class[cls] += conf
if cls in pred_class_count:
pred_class_count[cls] += 1
else:
pred_class_count[cls] = 1
for pred_class in pred_class_count.keys():
avg_conf_per_class[pred_class] = avg_conf_per_class[
pred_class] / pred_class_count[pred_class]
boxes = {
"predictions": {
"box_data": box_data,
"class_labels": names
}
} # inference-space
id = self.val_table_path_map[Path(path).name]
self.result_table.add_data(
self.current_epoch, id, self.val_table.data[id][1],
wandb.Image(self.val_table.data[id][1],
boxes=boxes,
classes=class_set), *avg_conf_per_class)
def _make_wandb_table():
classes = wandb.Classes([
{"id": 1, "name": "tree"},
{"id": 2, "name": "car"},
{"id": 3, "name": "road"},
])
table = wandb.Table(
columns=columns,
data=[
[1, 1, "string1", True, 1, 1.1, _make_wandb_image(), pc1, _make_html(), vid1, b1, aud1],
[2, 2, "string2", True, 1, 1.2, _make_wandb_image(), pc2, _make_html(), vid2, b2, aud_ref_https],
[3, 1, "string3", False, -0, -1.3, _make_wandb_image("2"), pc3, _make_html(), vid3, b3, aud_ref_s3],
[4, 3, "string4", False, -0, -1.4, _make_wandb_image("2"), pc4, _make_html(), vid4, b4, aud_ref_gs],
],
)
table.cast("class_id", classes.get_type())
return table
def _make_wandb_table():
classes = wandb.Classes([
{
"id": 1,
"name": "tree"
},
{
"id": 2,
"name": "car"
},
{
"id": 3,
"name": "road"
},
])
table = wandb.Table(
columns=columns,
data=[
[
1, "string", True, 1, 1.4,
_make_wandb_image(), pc1,
_make_html(), vid1
],
[
2, "string", True, 1, 1.4,
_make_wandb_image(), pc2,
_make_html(), vid2
],
[
1, "string2", False, -0, -1.4,
_make_wandb_image("2"), pc3,
_make_html(), vid3
],
[
3, "string2", False, -0, -1.4,
_make_wandb_image("2"), pc4,
_make_html(), vid4
],
],
)
table.cast("class_id", classes.get_type())
return table
def test_add_obj_wbimage_classes_obj_already_added(runner):
test_folder = os.path.dirname(os.path.realpath(__file__))
im_path = os.path.join(test_folder, "..", "assets", "2x2.png")
with runner.isolated_filesystem():
artifact = wandb.Artifact(type="dataset", name="my-arty")
classes = wandb.Classes([{"id": 0, "name": "person"}])
artifact.add(classes, "my-classes")
wb_image = wandb.Image(im_path, classes=classes)
artifact.add(wb_image, "my-image")
manifest = artifact.manifest.to_manifest_json()
if os.name == "nt": # windows
assert manifest["contents"] == {
"my-classes.classes.json": {
"digest": "eG00DqdCcCBqphilriLNfw==",
"size": 64,
},
"media\\images\\2x2.png": {
"digest": "L1pBeGPxG+6XVRQk4WuvdQ==",
"size": 71,
},
"my-image.image-file.json": {
"digest": "9pCnyQxcBiuNIEzlB0nEYw==",
"size": 209,
},
}
else:
assert manifest["contents"] == {
"my-classes.classes.json": {
"digest": "eG00DqdCcCBqphilriLNfw==",
"size": 64,
},
"media/images/2x2.png": {
"digest": "L1pBeGPxG+6XVRQk4WuvdQ==",
"size": 71,
},
"my-image.image-file.json": {
"digest": "jhtqSTpnbQyr2sL775eEkQ==",
"size": 207,
},
}
def log_training_progress(self, predn, path, names):
if self.val_table and self.result_table:
class_set = wandb.Classes([{
'id': id,
'name': name
} for id, name in names.items()])
box_data = []
total_conf = 0
for *xyxy, conf, cls in predn.tolist():
if conf >= 0.25:
box_data.append({
"position": {
"minX": xyxy[0],
"minY": xyxy[1],
"maxX": xyxy[2],
"maxY": xyxy[3]
},
"class_id":
int(cls),
"box_caption":
"%s %.3f" % (names[cls], conf),
"scores": {
"class_score": conf
},
"domain":
"pixel"
})
total_conf = total_conf + conf
boxes = {
"predictions": {
"box_data": box_data,
"class_labels": names
}
} # inference-space
id = self.val_table_map[Path(path).name]
self.result_table.add_data(
self.current_epoch, id, self.val_table.data[id][1],
wandb.Image(self.val_table.data[id][1],
boxes=boxes,
classes=class_set),
total_conf / max(1, len(box_data)))
def test_add_obj_wbimage_classes_obj(runner):
test_folder = os.path.dirname(os.path.realpath(__file__))
im_path = os.path.join(test_folder, "..", "assets", "2x2.png")
with runner.isolated_filesystem():
artifact = wandb.Artifact(type="dataset", name="my-arty")
classes = wandb.Classes([{"id": 0, "name": "person"}])
wb_image = wandb.Image(im_path, classes=classes)
artifact.add(wb_image, "my-image")
manifest = artifact.manifest.to_manifest_json()
if os.name == "nt": # windows
assert manifest["contents"] == {
"media\\cls.classes.json": {
"digest": "eG00DqdCcCBqphilriLNfw==",
"size": 64,
},
"media\\images\\2x2.png": {
"digest": "L1pBeGPxG+6XVRQk4WuvdQ==",
"size": 71,
},
"my-image.image-file.json": {
"digest": "omcGTjTrCSnwAfucXfPRsg==",
"size": 209,
},
}
else:
assert manifest["contents"] == {
"media/cls.classes.json": {
"digest": "eG00DqdCcCBqphilriLNfw==",
"size": 64,
},
"media/images/2x2.png": {
"digest": "L1pBeGPxG+6XVRQk4WuvdQ==",
"size": 71,
},
"my-image.image-file.json": {
"digest": "09JETFEpiuqBeICi09cY4A==",
"size": 206,
},
}
def _make_wandb_image(suffix=""):
class_labels = {1: "tree", 2: "car", 3: "road"}
test_folder = os.path.dirname(os.path.realpath(__file__))
im_path = os.path.join(test_folder, "..", "assets",
"test{}.png".format(suffix))
return wandb.Image(
im_path,
classes=wandb.Classes([
{
"id": 1,
"name": "tree"
},
{
"id": 2,
"name": "car"
},
{
"id": 3,
"name": "road"
},
]),
boxes={
"predictions": {
"box_data": [
{
"position": {
"minX": 0.1,
"maxX": 0.2,
"minY": 0.3,
"maxY": 0.4,
},
"class_id": 1,
"box_caption": "minMax(pixel)",
"scores": {
"acc": 0.1,
"loss": 1.2
},
},
{
"position": {
"minX": 0.1,
"maxX": 0.2,
"minY": 0.3,
"maxY": 0.4,
},
"class_id": 2,
"box_caption": "minMax(pixel)",
"scores": {
"acc": 0.1,
"loss": 1.2
},
},
],
"class_labels":
class_labels,
},
"ground_truth": {
"box_data": [
{
"position": {
"minX": 0.1,
"maxX": 0.2,
"minY": 0.3,
"maxY": 0.4,
},
"class_id": 1,
"box_caption": "minMax(pixel)",
"scores": {
"acc": 0.1,
"loss": 1.2
},
},
{
"position": {
"minX": 0.1,
"maxX": 0.2,
"minY": 0.3,
"maxY": 0.4,
},
"class_id": 2,
"box_caption": "minMax(pixel)",
"scores": {
"acc": 0.1,
"loss": 1.2
},
},
],
"class_labels":
class_labels,
},
},
masks={
"predictions": {
"mask_data": np.random.randint(0, 4, size=(30, 30)),
"class_labels": class_labels,
},
"ground_truth": {
"path": im_path,
"class_labels": class_labels
},
},
)
def main():
try:
# Download the data if not already
download_data()
# Initialize the run
with wandb.init(
project=WANDB_PROJECT, # The project to register this Run to
job_type=
"create_dataset", # The type of this Run. Runs of the same type can be grouped together in the UI
config=
{ # Custom configuration parameters which you might want to tune or adjust for the Run
"num_examples":
NUM_EXAMPLES, # The number of raw samples to include.
"scale_factor": 2 # The scaling factor for the images
}) as run:
# Setup a WandB Classes object. This will give additional metadata for visuals
class_set = wandb.Classes([{
'name': name,
'id': id
} for name, id in zip(BDD_CLASSES, BDD_IDS)])
# Setup a WandB Table object to hold our dataset
table = wandb.Table(columns=[
"id", "train_image", "colored_image", "label_mask",
"dominant_class"
])
# Fill up the table
for ndx in range(run.config["num_examples"]):
# First, we will build a wandb.Image to act as our raw example object
# classes: the classes which map to masks and/or box metadata
# masks: the mask metadata. In this case, we use a 2d array where each cell corresponds to the label (this comes directlyfrom the dataset)
# boxes: the bounding box metadata. For example sake, we create bounding boxes by looking at the mask data and creating boxes which fully encolose each class.
# The data is an array of objects like:
# "position": {
# "minX": minX,
# "maxX": maxX,
# "minY": minY,
# "maxY": maxY,
# },
# "class_id" : id_num,
# }
example = wandb.Image(
get_scaled_train_image(ndx, run.config.scale_factor),
classes=class_set,
masks={
"ground_truth": {
"mask_data":
get_scaled_mask_label(ndx, run.config.scale_factor)
},
},
boxes={
"ground_truth": {
"box_data":
get_scaled_bounding_boxes(ndx,
run.config.scale_factor)
}
})
# Next, we create two additional images which may be helpful during analysis. Notice that the additional metadata is optional.
color_label = wandb.Image(
get_scaled_color_mask(ndx, run.config.scale_factor))
label_mask = wandb.Image(
get_scaled_mask_label(ndx, run.config.scale_factor))
# Finally, we add a row of our newly constructed data.
table.add_data(train_ids[ndx], example, color_label,
label_mask, get_dominant_class(label_mask))
# Create an Artifact (versioned folder)
artifact = wandb.Artifact(name="raw_data", type="dataset")
# add the table to the artifact
artifact.add(table, "raw_examples")
# Finally, log the artifact
run.log_artifact(artifact)
print("Step 1/5 Complete")
# This step should look familiar by now:
with wandb.init(project=WANDB_PROJECT,
job_type="split_dataset",
config={
"train_pct": 0.7,
}) as run:
# Get the latest version of the artifact. Notice the name alias follows this convention: "<ARTIFACT_NAME>:<VERSION>"
# when version is set to "latest", then the latest version will always be used. However, you can pin to a version by
# using an alias such as "raw_data:v0"
dataset_artifact = run.use_artifact("raw_data:latest")
# Next, we "get" the table by the same name that we saved it in the last run.
data_table = dataset_artifact.get("raw_examples")
# Now we can build two separate artifacts for later use. We will first split the raw table into two parts,
# then create two different artifacts, each of which will hold our new tables. We create two artifacts so that
# in future runs, we can selectively decide which subsets of data to download.
# Create the tables
train_count = int(len(data_table.data) * run.config.train_pct)
train_table = wandb.Table(columns=data_table.columns,
data=data_table.data[:train_count])
test_table = wandb.Table(columns=data_table.columns,
data=data_table.data[train_count:])
# Create the artifacts
train_artifact = wandb.Artifact("train_data", "dataset")
test_artifact = wandb.Artifact("test_data", "dataset")
# Save the tables to the artifacts
train_artifact.add(train_table, "train_table")
test_artifact.add(test_table, "test_table")
# Log the artifacts out as outputs of the run
run.log_artifact(train_artifact)
run.log_artifact(test_artifact)
print("Step 2/5 Complete")
# Again, create a run.
with wandb.init(project=WANDB_PROJECT, job_type="model_train") as run:
# Similar to before, we will load in the artifact and asset we need. In this case, the training data
train_artifact = run.use_artifact("train_data:latest")
train_table = train_artifact.get("train_table")
# Next, we split out the labels and train the model
train_data, mask_data = make_datasets(train_table, n_classes)
model = ExampleSegmentationModel(n_classes)
model.train(train_data, mask_data)
# Finally we score the model. Behind the scenes, we score each mask on it's IOU score.
scores, results = score_model(model, train_data, mask_data,
n_classes)
# Let's create a new table. Notice that we create many columns - an evaluation score for each class type.
results_table = wandb.Table(
columns=["id", "pred_mask", "dominant_pred"] + BDD_CLASSES,
# Data construction is similar to before, but we now use the predicted masks and bound boxes.
data=[[
train_table.data[ndx][0],
wandb.Image(train_table.data[ndx][1],
masks={
"train_predicted_truth": {
"mask_data": results[ndx],
},
},
boxes={
"ground_truth": {
"box_data": mask_to_bounding(
results[ndx])
}
}),
BDD_CLASSES[get_dominant_id_ndx(results[ndx])],
] + list(row) for ndx, row in enumerate(scores)])
# We create an artifact, add the table, and log it as part of the run.
results_artifact = wandb.Artifact("train_results", "dataset")
results_artifact.add(results_table, "train_iou_score_table")
run.log_artifact(results_artifact)
# Finally, let's save the model as a flat file and add that to it's own artifact.
model.save("model.pkl")
model_artifact = wandb.Artifact("trained_model", "model")
model_artifact.add_file("model.pkl")
run.log_artifact(model_artifact)
print("Step 3/5 Complete")
with wandb.init(project=WANDB_PROJECT, job_type="model_eval") as run:
# Retrieve the test data
test_artifact = run.use_artifact("test_data:latest")
test_table = test_artifact.get("test_table")
test_data, mask_data = make_datasets(test_table, n_classes)
# Download the saved model file.
model_artifact = run.use_artifact("trained_model:latest")
path = model_artifact.get_path("model.pkl").download()
# Load the model from the file and score it
model = ExampleSegmentationModel.load(path)
scores, results = score_model(model, test_data, mask_data,
n_classes)
# Create a predicted score table similar to step 3.
results_artifact = wandb.Artifact("test_results", "dataset")
data = [[
test_table.data[ndx][0],
wandb.Image(test_table.data[ndx][1],
masks={
"test_predicted_truth": {
"mask_data": results[ndx],
},
},
boxes={
"ground_truth": {
"box_data": mask_to_bounding(results[ndx])
}
}),
BDD_CLASSES[get_dominant_id_ndx(results[ndx])],
] + list(row) for ndx, row in enumerate(scores)]
# And log out the results.
results_artifact.add(
wandb.Table(["id", "pred_mask_test", "dominant_pred_test"] +
BDD_CLASSES,
data=data), "test_iou_score_table")
run.log_artifact(results_artifact)
print("Step 4/5 Complete")
with wandb.init(project=WANDB_PROJECT,
job_type="model_result_analysis") as run:
# Retrieve the original raw dataset
dataset_artifact = run.use_artifact("raw_data:latest")
data_table = dataset_artifact.get("raw_examples")
# Retrieve the train and test score tables
train_artifact = run.use_artifact("train_results:latest")
train_table = train_artifact.get("train_iou_score_table")
test_artifact = run.use_artifact("test_results:latest")
test_table = test_artifact.get("test_iou_score_table")
# Join the tables on ID column and log them as outputs.
train_results = wandb.JoinedTable(train_table, data_table, "id")
test_results = wandb.JoinedTable(test_table, data_table, "id")
artifact = wandb.Artifact("summary_results", "dataset")
artifact.add(train_results, "train_results")
artifact.add(test_results, "test_results")
run.log_artifact(artifact)
print("Step 5/5 Complete")
if WANDB_PROJECT_ENV is not None:
os.environ["WANDB_PROJECT"] = WANDB_PROJECT_ENV
if WANDB_SILENT_ENV is not None:
os.environ["WANDB_SILENT"] = WANDB_SILENT_ENV
finally:
cleanup()
from mmcv.runner import load_checkpoint
from mmcv.parallel import MMDataParallel
from pycocotools.coco import COCO
import os
import cv2
import wandb
import numpy as np
import matplotlib.pyplot as plt
classes = ("UNKNOWN", "General trash", "Paper", "Paper pack", "Metal", "Glass",
"Plastic", "Styrofoam", "Plastic bag", "Battery", "Clothing")
class_id_to_label = {v: k for v, k in enumerate(classes)}
class_set = wandb.Classes([{
'name': name,
'id': id
} for id, name in class_id_to_label.items()])
class_num = 11
def get_data_and_output(cfg, checkpoint_path, mode):
PREFIX = '../../input/data/'
if mode == "test":
data = cfg.data.test
elif mode == "train":
data = cfg.data.train
elif mode == "val":
data = cfg.data.val
else:
raise ValueError('mode can have train, val or test')
# Initialize the run
with wandb.init(
project=WANDB_PROJECT, # The project to register this Run to
job_type=
"create_dataset", # The type of this Run. Runs of the same type can be grouped together in the UI
config=
{ # Custom configuration parameters which you might want to tune or adjust for the Run
"num_examples":
NUM_EXAMPLES, # The number of raw samples to include.
"scale_factor": 2 # The scaling factor for the images
}) as run:
# Setup a WandB Classes object. This will give additional metadata for visuals
class_set = wandb.Classes([{
'name': name,
'id': id
} for name, id in zip(BDD_CLASSES, BDD_IDS)])
# Setup a WandB Table object to hold our dataset
table = wandb.Table(columns=[
"id", "train_image", "colored_image", "label_mask",
"dominant_class"
])
# Fill up the table
for ndx in range(run.config["num_examples"]):
# First, we will build a wandb.Image to act as our raw example object
# classes: the classes which map to masks and/or box metadata
# masks: the mask metadata. In this case, we use a 2d array where each cell corresponds to the label (this comes directlyfrom the dataset)
# boxes: the bounding box metadata. For example sake, we create bounding boxes by looking at the mask data and creating boxes which fully encolose each class.
