def check_and_load(path: str, remote_url: str = None) -> None:
""" Check if the file at the path exists. If not, and remote_url
is not None, then offer to load it. """
if not os.path.exists(path):
print("\nError: File {} does not exist.".format(path))
if remote_url is not None:
while True:
response = input(
"Do you want to load it from {}? [y/n]: ".format(
remote_url))
if response == "y":
print(
"Loading, please wait! (The file is pretty big so this could take a while...)"
)
pather.create(path)
urllib.request.urlretrieve(remote_url, path)
Logger.log_field("Successfully Loaded", path)
if response == "n":
break
print("Response is not valid. Please enter y or n.\n")
raise Exception("File not found, unable to proceed.")
def _eval(self, units: List[SCCUnit]):
correct = 0
for unit in units:
result = self.net.process(unit.image)
if result == unit.type:
correct += 1
Logger.log_field("Accuracy", "{:.1f}%".format(100 * (correct/len(units))))
def image(self):
""" Get the CV2 image for this sample (BGR Format). """
if not self.is_locally_loaded:
self.load()
try:
return cv2.imread(self._local_image_path)
except Exception as e:
Logger.log_field(f"Error Loading Image {self.key}", e)
os.remove(self._local_image_path)
exit(1)
def create_samples(self, path) -> Dict[str, Sample]:
""" Create samples from the rows in the image URL CSV. """
samples: Dict[str, Sample] = {}
def action(row):
sample = Sample()
sample.key = row[0].split(".")[0] # Remove the .jpg extension.
sample.remote_path = row[1]
samples[sample.key] = sample
self.execute_on_csv(path, action)
Logger.log_field("Samples Loaded", len(samples))
return samples
def process(self, input_path):
units = []
folders = os.listdir(input_path)
for folder in folders:
type = Cell.NEUTRAL
if folder == "positive":
type = Cell.POSITIVE
if folder == "negative":
type = Cell.NEGATIVE
folder_path = os.path.join(input_path, folder)
files = os.listdir(folder_path)
for file in files:
unit = SCCUnit()
unit.image_path = os.path.join(folder_path, file)
unit.type = type
units.append(unit)
Logger.log_field("Units Created", len(units))
Logger.log_field("Positive", sum(1 for unit in units if unit.type == Cell.POSITIVE))
Logger.log_field("Negative", sum(1 for unit in units if unit.type == Cell.NEGATIVE))
Logger.log_field("Neutral", sum(1 for unit in units if unit.type == Cell.NEUTRAL))
self._train(units)
self._eval(units)
self._save("output/scc.pt")
def scan(self, image_path, label_path):
image = cv2.imread(image_path)
Logger.log_field("Image Loaded Size", image.shape)
cells = []
# Read the CSV labels.
with open(label_path) as file:
reader = csv.reader(file, delimiter=",")
skip_row = True
for row in reader:
# Skip the header.
if skip_row:
skip_row = False
continue
region_shape = json.loads(row[5])
# region_attributes = json.loads(row[6])
cell = Cell()
cell.x_points = region_shape["all_points_x"]
cell.y_points = region_shape["all_points_y"]
cell.calibrate_points()
cells.append(cell)
overlay = np.zeros_like(image)
colors = self.random_colors(len(cells))
i = 0
for cell in cells:
cv2.fillConvexPoly(overlay, cell.points, colors[i] * 0.2)
cv2.polylines(overlay, [cell.points], True, colors[i], 24)
i += 1
pass
compose = cv2.addWeighted(image, 1.0, overlay, 1.0, 0.0)
compose = cv2.resize(compose, (512, 512))
overlay = cv2.resize(overlay, (512, 512))
final = np.zeros((512, 1024, 3), dtype=np.uint8)
final[:512, :512] = compose
final[:512, 512:] = overlay
cv2.imwrite("input/sample.png", final)
def execute_on_csv(path: str,
action: classmethod,
skip_first_row: bool = False):
""" Run a specified action on each row of the CSV file. """
with open(path) as f:
row_count = sum(1 for _ in f)
f.seek(0)
reader = csv.reader(f, delimiter=",")
Logger.log_field("Reading CSV", os.path.split(path)[1])
Logger.log_field("Rows", row_count)
for row in reader:
# Skip the first row - it is just labels.
if skip_first_row:
skip_first_row = False
continue
action(row)
def display_stats(instances: Dict[str, int],
title: str = "SOMETHING",
file_name: str = "graph_name",
n_display: int = 20):
sorted_instances = sorted(instances.items(), key=lambda kv: kv[1])
sorted_instances.reverse()
Logger.log_special(title, with_gap=True)
for i in range(n_display):
Logger.log_field(
loader.get_label(sorted_instances[i][0]).upper(),
sorted_instances[i][1])
# Create the figure.
plt.style.use("ggplot")
fig, ax = plt.subplots(figsize=(15, 8))
short_instances = sorted_instances[:n_display]
# Labels.
y_label = [loader.get_label(s[0]) for s in short_instances]
y_label.insert(0, "(OTHERS)")
y = np.arange(len(y_label))
# Values.
x = [s[1] for s in short_instances]
x.insert(0, sum([s[1] for s in sorted_instances[n_display:]]))
c_map = plt.get_cmap("plasma")
colors = c_map(1 - y / len(y))
colors[0] = (0.7, 0.7, 0.7, 1.0)
# Plot the graph.
plt.barh(y, x, height=0.5, color=colors)
ax.set_yticks(y)
ax.set_yticklabels(y_label)
ax.invert_yaxis()
ax.set_title(f"{title}: ({len(samples)} Images)")
ax.set_xlabel("Count")
ax.set_ylabel("Class Name")
plt.savefig(f"{settings.OUTPUT_DIRECTORY}/{file_name}.png")
plt.clf()
def load(self):
""" Load the image for this sample into the designated storage file."""
Logger.log_field("Loading Image", self.key)
try:
pather.create(self._local_image_path)
urllib.request.urlretrieve(self.remote_path, self._local_image_path)
except Exception as e:
Logger.log_field("Loading Failed", e)
Logger.log_field("Loading Complete", self.key)
def cycle_predict(self, image, equalizer_mask=None):
prediction = Prediction()
extract_image = np.copy(image)
extract_mask = np.ones_like(equalizer_mask, dtype=np.bool)
width = image.shape[1]
height = image.shape[0]
max_volume = self._get_max_volume(height, width)
attempts = 0
while attempts < self.K_MAX_CYCLES:
attempts += 1
units, extract_image, extract_mask = self.predict_and_extract(
extract_image, extract_mask, max_volume)
if len(units) > 0:
for unit in units:
prediction.units.append(unit)
else:
break
Logger.log_field("Prediction Attempts", attempts)
Logger.log_field("Total Cells Predicted", len(prediction.units))
return prediction
from modules.data.slide import Slide
from modules.scanner import Scanner
from tools.util import pather
from tools.util.logger import Logger
__author__ = "Jakrin Juangbhanich"
__email__ = "*****@*****.**"
__version__ = "0.0.0"
parser = argparse.ArgumentParser(description='<Script Info>')
parser.add_argument('-i', '--input', default='test', type=str, help="<help>")
parser.add_argument('-o', '--output', default='batch1', type=str, help="<help>")
parser.add_argument('-f', '--flag', action="store_true", help="<help>")
args = parser.parse_args()
if __name__ == "__main__":
input_path = os.path.join("input", args.input)
output_path = os.path.join("output", args.output)
pather.create(output_path, clear=True)
Logger.log_header("Running Scanner", with_gap=True)
Logger.log_field("Version", __version__)
Logger.log_field("Input Path", input_path)
Logger.log_field("Input Path", output_path)
scanner = Scanner(output_path=output_path)
slides: List[Slide] = loader.load_testing_slides(input_path)
scanner.process(slides)
args = get_args()
set_index = args.set_index
sample_count = args.sample_count
if __name__ == "__main__":
# Load the project settings and required modules.
Logger.log_special("Running Sample Loader", with_gap=True)
settings = ProjectSettings("settings.yaml")
# Load the label mapping.
loader = Loader()
loader.load_labels(settings.LABELS_FILE)
Logger.log_field("Labels Loaded", len(loader.label_map))
# Load the samples from the set that we want.
samples = Loader.load_sample_set(set_index)
loaded_samples = [
s for s in samples
if (s.is_locally_loaded and len(s.detect_regions) > 0)
]
# How many samples loaded?
n_loaded_samples = len(loaded_samples)
Logger.log_field("Samples with Images", n_loaded_samples)
if n_loaded_samples == 0:
raise Exception(
"None of the samples in this set have been downloaded! "
"Please run cmd_load_sample_images first for this set.")
"""
from __future__ import print_function
from __future__ import division
from __future__ import absolute_import
import argparse
import os
from modules.ai.single_cell_classifier.scc_trainer import SCCTrainer
from tools.util.logger import Logger
__author__ = "Jakrin Juangbhanich"
__email__ = "*****@*****.**"
__version__ = "0.0.1"
parser = argparse.ArgumentParser(description='<Script Info>')
parser.add_argument('-i', '--input', default='data', type=str, help="<help>")
parser.add_argument('-f', '--flag', action="store_true", help="<help>")
args = parser.parse_args()
if __name__ == "__main__":
input_path = os.path.join("input", args.input)
Logger.log_header("Running Cell-Scan Trainer", with_gap=True)
Logger.log_field("Version", __version__)
Logger.log_field("Input Folder", input_path)
scc_trainer = SCCTrainer()
scc_trainer.process(input_path)
args = get_args()
set_index = args.set_index
max_threads = args.max_threads
if __name__ == "__main__":
# Load the project settings and required modules.
Logger.log_special("Running Sample Loader", with_gap=True)
settings = ProjectSettings("settings.yaml")
set_path = os.path.join(settings.SAMPLES_DIRECTORY,
f"sample_set_{set_index}.json")
if not os.path.exists(set_path):
Logger.log_field(
"Error",
"No file found at {}. Have you created the samples using cmd_create_samples yet?"
)
exit(1)
Logger.log_special("Begin Sample Image Download", with_gap=True)
samples = Loader.load_sample_set_from_file(set_path)
unloaded_samples = [s for s in samples if not s.is_locally_loaded]
n_unloaded_samples = len(unloaded_samples)
n_samples = len(samples)
Logger.log_field("Samples Loaded",
"{}/{}".format(n_samples - n_unloaded_samples, n_samples))
i = 0
for sample in unloaded_samples:
while True: