valErrors += 1
print("\nErrors:")
print("Train: {}\nVal:\t{}".format(trainErrors, valErrors))
print("\nNaNs:")
print("Train: {}\nVal:\t{}".format(np.sum(trainIndex['rede3'].isna()),
np.sum(valIndex['rede3'].isna())))
trainIndex.dropna(axis=0, subset=["HashMD5"], inplace=True)
valIndex.dropna(axis=0, subset=["HashMD5"], inplace=True)
print("\nNaNs:")
print("Train: {}\nVal:\t{}".format(np.sum(trainIndex['rede3'].isna()),
np.sum(valIndex['rede3'].isna())))
dutils.df_to_csv(trainIndex, trainPath)
dutils.df_to_csv(valIndex, valPath)
input("\nMoving datasets to train folder.\nPress enter to continue.\n")
# Move dataset to training folder, split in train/val folders
dutils.copy_dataset_to_folder(trainPath,
semiautoDatasetPath / "train",
path_column="FramePath")
dutils.move_to_class_folders(trainPath,
semiautoDatasetPath / "train",
target_net="rede3",
target_class=None,
move=True)
dutils.copy_dataset_to_folder(valPath,
strTotal += ("\nTotal: {}\t({:.2f} %)".format(
totalNeg + totalPos,
(totalNeg + totalPos) / (totalLen + previousManualLen) * 100))
print(strAuto)
print(strManual)
print(strTotal)
utils.write_string(strAuto + strManual + strTotal, reportPath, mode='w')
# Convert final_annotated_images to a net evaluation setup and copy to dirs.iter_folder
# Translate rede1 3 classes to 2 for binary classification
# Get only past level positive examples for rede2
# Get only positive class examples for rede3 (in other script)
netName = "rede" + str(rede)
if rede == 3:
print(
"\nPlease run script \'run/fuse_binary_datasets.py\' to generate evaluation dataset."
)
exit()
elif rede == 2:
# Get only positive examples from previous level
mask = (annotatedIndexFull['rede1'] == commons.rede1_positive)
evalDataset = annotatedIndexFull.loc[mask, :]
elif rede == 1:
evalDataset = annotatedIndexFull.copy()
# evalDataset.dropna(subset=[netName], inplace=True)
# Translate to binary classes
evalDataset[netName] = dutils.translate_labels(evalDataset[netName], netName)
dutils.df_to_csv(evalDataset, binaryDatasetPath)
print("train set: {} images.".format(len(trainIndex)))
print("val set: {} images.".format(len(valIndex)))
print("\nErrors:")
print("Train: {}\nVal:\t{}".format(trainErrors, valErrors))
print("\nNaNs:")
print("Train: {}\nVal:\t{}".format(np.sum(trainIndex[netName].isna()),
np.sum(valIndex[netName].isna())))
trainIndex.dropna(axis=0, subset=["HashMD5"], inplace=True)
valIndex.dropna(axis=0, subset=["HashMD5"], inplace=True)
print("\nNaNs:")
print("Train: {}\nVal:\t{}".format(np.sum(trainIndex[netName].isna()),
np.sum(valIndex[netName].isna())))
dutils.df_to_csv(trainIndex, trainTempPath)
dutils.df_to_csv(valIndex, valTempPath)
input("\nMoving datasets to train folder.\nPress enter to continue.\n")
# Move dataset to training folder, split in train/val folders
dutils.copy_dataset_to_folder(trainTempPath,
semiautoDatasetPath / "train",
path_column="FramePath")
dutils.move_to_class_folders(trainTempPath,
semiautoDatasetPath / "train",
target_net=netName,
target_class=None,
move=True)
dutils.copy_dataset_to_folder(valTempPath,
semiautoDatasetPath / "val",
print("rede " + str(rede))
classFolder = Path(classFolder)
# targetClass = str(classFolder.name).split("_")[-1]
print("\nProcessing class ", classFolder.name)
errorDf = pd.read_csv(classFolder / "sampled_images_corrections.csv")
autoDf = pd.read_csv(classFolder / "automatic_check_index.csv")
print("Calculating hashes...")
errorDf["FrameHash"] = utils.compute_file_hash_list(
errorDf.loc[:, "imagem"],
folder=(classFolder / "automatic") / "sampled_images")
dutils.df_to_csv(errorDf,
classFolder / "sampled_images_corrections_results.csv",
verbose=False)
autoDf.set_index("FrameHash", inplace=True, drop=False)
errorDf.set_index("FrameHash", inplace=True, drop=False)
errorDf['error_check'] = [None] * len(errorDf)
for i in autoDf.index:
errorTag = str(errorDf.loc[i, 'rede' + rede]).lower()
autoTag = str(autoDf.loc[i, 'rede' + rede]).lower()
if (autoTag == targetClass) and (autoTag == errorTag):
errorCheck = False
elif (autoTag != targetClass) and (errorTag != targetClass):
errorCheck = False
else:
errorCheck = True