def parse_model_config(path):
"""Parse the yolov3 layer configuration file and returns module definitions."""
lines = utils.get_lines(path)
module_defs = []
for line in lines:
if line.startswith("["): # This marks the start of a new block
module_defs.append({})
module_defs[-1]["type"] = line[1:-1].rstrip()
if module_defs[-1]["type"] == "convolutional":
module_defs[-1]["batch_normalize"] = 0
else:
key, value = line.split("=")
value = value.strip()
module_defs[-1][key.rstrip()] = value.strip()
return module_defs
def load_splits(self, folder):
"""Load previously-generated text file splits into the current folder if found.
Returns a boolean value indicating if the operation was successful.
"""
split_paths = [
f"{folder}/{self.prefix}_{name}.txt" for name in self.sets
]
if all(os.path.exists(path) for path in split_paths):
file_lists = [get_lines(path) for path in split_paths]
labeled_sets = self.convert_splits(file_lists)
for name, split_set in zip(self.sets, labeled_sets):
setattr(self, name, split_set)
return True
return False
def get_batch_test_set(config, reserve_batches):
out_dir = config["output"]
num_classes = len(utils.load_classes(config["class_list"]))
batch_sets = sorted(glob.glob(f"{out_dir}/sample*.txt"),
key=utils.get_sample)
test_imgs = list()
batches_removed = 0
for batch_set in reversed(batch_sets):
imgs = utils.get_lines(batch_set)
if len(imgs) < config["sampling_batch"] or reserve_batches != 0:
test_imgs += imgs
batches_removed += 1
if not (len(imgs) < config["sampling_batch"]):
reserve_batches -= 1
return LabeledSet(test_imgs, num_classes), batches_removed
def get_batch_splits(self, batch_size, output):
"""Output text files lists of random image batches of the specified size.
Does not stratify by class, and the last batch may have fewer images than desired.
"""
batches = math.ceil(len(self) / batch_size)
batch_files = [
f"{output}/{self.prefix}{i}.txt" for i in range(batches)
]
if all(os.path.exists(path) for path in batch_files):
batch_splits = [get_lines(path) for path in batch_files]
print("Previous batch splits found")
else:
batch_splits = self.split_batch(batch_size)
for i, path in enumerate(batch_files):
with open(path, "w+") as out:
out.write("\n".join(batch_splits[i]))
return self.convert_splits(batch_splits)
def get_test_sets(config, prefix):
out_dir = config["output"]
num_classes = len(utils.load_classes(config["class_list"]))
epoch_splits = utils.get_epoch_splits(config, prefix)
# Initial test set
init_test_set = f"{out_dir}/init_test.txt"
init_test_folder = LabeledSet(init_test_set, num_classes)
# Only data from the (combined) iteration test sets (75% sampling + 25% seen data)
iter_test_sets = [
f"{out_dir}/{prefix}{i}_test.txt" for i in range(len(epoch_splits))
]
iter_img_files = list()
for file in iter_test_sets:
iter_img_files += utils.get_lines(file)
all_iter_sets = LabeledSet(iter_img_files, num_classes)
# Test sets filtered for only sampled images
sampled_imgs = [
img for img in iter_img_files if config["sample_set"] in img
]
sample_test = LabeledSet(sampled_imgs, num_classes)
# Data from all test sets
all_test = LabeledSet(sampled_imgs, num_classes)
all_test += init_test_folder
test_sets = {
"init": init_test_folder,
"all_iter": all_iter_sets,
"sample": sample_test,
"all": all_test,
}
if prefix != "init":
for i in range(len(epoch_splits)):
test_sets[f"cur_iter{i + 1}"] = LabeledSet(iter_test_sets[i],
num_classes)
return test_sets
def load_data(output, by_actual=True, add_all=True, filter=None, conf_thresh=0.5):
samples = dict()
all_data = list()
actual = list()
pred = list()
with open(output, newline="\n") as csvfile:
reader = csv.DictReader(csvfile)
if filter is not None:
filter_list = utils.get_lines(filter)
for row in reader:
if filter is not None and row["file"] not in filter_list:
continue
actual.append(row["actual"])
pred.append(row["detected"])
all_data.append(row)
key_val = (
row["actual"]
if by_actual or row["detected"] == str()
else row["detected"]
)
if key_val in samples.keys():
samples[key_val].append(row)
else:
samples[key_val] = [row]
samples = {k: samples[k] for k in sorted(samples)}
results = [ClassResults(k, v, conf_thresh=conf_thresh) for k, v in samples.items()]
mat = confusion_matrix(actual, pred, labels=list(samples.keys()) + [""])
if add_all:
results.append(ClassResults("All", all_data, conf_thresh=conf_thresh))
return results, mat
def __init__(self, src, img_size, prefix=str()):
"""
Parameters:
src (str): source of the image folder. Can be a list or set of image paths,
a text file of image paths, or a Darknet-labeled folder
img_size (int): square resolution to pad or downsize images to
prefix (str): string to represent the image folder. used in output filenames
"""
if isinstance(src, (list, set)):
self.imgs = set(src)
elif ".txt" in src:
if os.path.isfile(src):
self.imgs = get_lines(src)
else:
self.imgs = list()
print(f"{src} is an invalid file. Ignoring...")
elif os.path.isdir(src):
self.imgs = get_images(src)
else:
raise TypeError("ImageFolder source must be file list or folder")
self.prefix = prefix
self.img_size = img_size
def get_classes(self, label_path):
"""Get a list of classes from a Darknet label."""
labels = get_lines(label_path)
classes = [int(lab.split(" ")[0]) for lab in labels if lab != ""]
return [c for c in classes if c in range(self.num_classes)]