def check_and_load(path: str, remote_url: str = None) -> None: """ Check if the file at the path exists. If not, and remote_url is not None, then offer to load it. """ if not os.path.exists(path): print("\nError: File {} does not exist.".format(path)) if remote_url is not None: while True: response = input( "Do you want to load it from {}? [y/n]: ".format( remote_url)) if response == "y": print( "Loading, please wait! (The file is pretty big so this could take a while...)" ) pather.create(path) urllib.request.urlretrieve(remote_url, path) Logger.log_field("Successfully Loaded", path) if response == "n": break print("Response is not valid. Please enter y or n.\n") raise Exception("File not found, unable to proceed.")
def _eval(self, units: List[SCCUnit]): correct = 0 for unit in units: result = self.net.process(unit.image) if result == unit.type: correct += 1 Logger.log_field("Accuracy", "{:.1f}%".format(100 * (correct/len(units))))
def image(self): """ Get the CV2 image for this sample (BGR Format). """ if not self.is_locally_loaded: self.load() try: return cv2.imread(self._local_image_path) except Exception as e: Logger.log_field(f"Error Loading Image {self.key}", e) os.remove(self._local_image_path) exit(1)
def create_samples(self, path) -> Dict[str, Sample]: """ Create samples from the rows in the image URL CSV. """ samples: Dict[str, Sample] = {} def action(row): sample = Sample() sample.key = row[0].split(".")[0] # Remove the .jpg extension. sample.remote_path = row[1] samples[sample.key] = sample self.execute_on_csv(path, action) Logger.log_field("Samples Loaded", len(samples)) return samples
def process(self, input_path): units = [] folders = os.listdir(input_path) for folder in folders: type = Cell.NEUTRAL if folder == "positive": type = Cell.POSITIVE if folder == "negative": type = Cell.NEGATIVE folder_path = os.path.join(input_path, folder) files = os.listdir(folder_path) for file in files: unit = SCCUnit() unit.image_path = os.path.join(folder_path, file) unit.type = type units.append(unit) Logger.log_field("Units Created", len(units)) Logger.log_field("Positive", sum(1 for unit in units if unit.type == Cell.POSITIVE)) Logger.log_field("Negative", sum(1 for unit in units if unit.type == Cell.NEGATIVE)) Logger.log_field("Neutral", sum(1 for unit in units if unit.type == Cell.NEUTRAL)) self._train(units) self._eval(units) self._save("output/scc.pt")
def scan(self, image_path, label_path): image = cv2.imread(image_path) Logger.log_field("Image Loaded Size", image.shape) cells = [] # Read the CSV labels. with open(label_path) as file: reader = csv.reader(file, delimiter=",") skip_row = True for row in reader: # Skip the header. if skip_row: skip_row = False continue region_shape = json.loads(row[5]) # region_attributes = json.loads(row[6]) cell = Cell() cell.x_points = region_shape["all_points_x"] cell.y_points = region_shape["all_points_y"] cell.calibrate_points() cells.append(cell) overlay = np.zeros_like(image) colors = self.random_colors(len(cells)) i = 0 for cell in cells: cv2.fillConvexPoly(overlay, cell.points, colors[i] * 0.2) cv2.polylines(overlay, [cell.points], True, colors[i], 24) i += 1 pass compose = cv2.addWeighted(image, 1.0, overlay, 1.0, 0.0) compose = cv2.resize(compose, (512, 512)) overlay = cv2.resize(overlay, (512, 512)) final = np.zeros((512, 1024, 3), dtype=np.uint8) final[:512, :512] = compose final[:512, 512:] = overlay cv2.imwrite("input/sample.png", final)
def execute_on_csv(path: str, action: classmethod, skip_first_row: bool = False): """ Run a specified action on each row of the CSV file. """ with open(path) as f: row_count = sum(1 for _ in f) f.seek(0) reader = csv.reader(f, delimiter=",") Logger.log_field("Reading CSV", os.path.split(path)[1]) Logger.log_field("Rows", row_count) for row in reader: # Skip the first row - it is just labels. if skip_first_row: skip_first_row = False continue action(row)
def display_stats(instances: Dict[str, int], title: str = "SOMETHING", file_name: str = "graph_name", n_display: int = 20): sorted_instances = sorted(instances.items(), key=lambda kv: kv[1]) sorted_instances.reverse() Logger.log_special(title, with_gap=True) for i in range(n_display): Logger.log_field( loader.get_label(sorted_instances[i][0]).upper(), sorted_instances[i][1]) # Create the figure. plt.style.use("ggplot") fig, ax = plt.subplots(figsize=(15, 8)) short_instances = sorted_instances[:n_display] # Labels. y_label = [loader.get_label(s[0]) for s in short_instances] y_label.insert(0, "(OTHERS)") y = np.arange(len(y_label)) # Values. x = [s[1] for s in short_instances] x.insert(0, sum([s[1] for s in sorted_instances[n_display:]])) c_map = plt.get_cmap("plasma") colors = c_map(1 - y / len(y)) colors[0] = (0.7, 0.7, 0.7, 1.0) # Plot the graph. plt.barh(y, x, height=0.5, color=colors) ax.set_yticks(y) ax.set_yticklabels(y_label) ax.invert_yaxis() ax.set_title(f"{title}: ({len(samples)} Images)") ax.set_xlabel("Count") ax.set_ylabel("Class Name") plt.savefig(f"{settings.OUTPUT_DIRECTORY}/{file_name}.png") plt.clf()
def load(self): """ Load the image for this sample into the designated storage file.""" Logger.log_field("Loading Image", self.key) try: pather.create(self._local_image_path) urllib.request.urlretrieve(self.remote_path, self._local_image_path) except Exception as e: Logger.log_field("Loading Failed", e) Logger.log_field("Loading Complete", self.key)
def cycle_predict(self, image, equalizer_mask=None): prediction = Prediction() extract_image = np.copy(image) extract_mask = np.ones_like(equalizer_mask, dtype=np.bool) width = image.shape[1] height = image.shape[0] max_volume = self._get_max_volume(height, width) attempts = 0 while attempts < self.K_MAX_CYCLES: attempts += 1 units, extract_image, extract_mask = self.predict_and_extract( extract_image, extract_mask, max_volume) if len(units) > 0: for unit in units: prediction.units.append(unit) else: break Logger.log_field("Prediction Attempts", attempts) Logger.log_field("Total Cells Predicted", len(prediction.units)) return prediction
from modules.data.slide import Slide from modules.scanner import Scanner from tools.util import pather from tools.util.logger import Logger __author__ = "Jakrin Juangbhanich" __email__ = "*****@*****.**" __version__ = "0.0.0" parser = argparse.ArgumentParser(description='<Script Info>') parser.add_argument('-i', '--input', default='test', type=str, help="<help>") parser.add_argument('-o', '--output', default='batch1', type=str, help="<help>") parser.add_argument('-f', '--flag', action="store_true", help="<help>") args = parser.parse_args() if __name__ == "__main__": input_path = os.path.join("input", args.input) output_path = os.path.join("output", args.output) pather.create(output_path, clear=True) Logger.log_header("Running Scanner", with_gap=True) Logger.log_field("Version", __version__) Logger.log_field("Input Path", input_path) Logger.log_field("Input Path", output_path) scanner = Scanner(output_path=output_path) slides: List[Slide] = loader.load_testing_slides(input_path) scanner.process(slides)
args = get_args() set_index = args.set_index sample_count = args.sample_count if __name__ == "__main__": # Load the project settings and required modules. Logger.log_special("Running Sample Loader", with_gap=True) settings = ProjectSettings("settings.yaml") # Load the label mapping. loader = Loader() loader.load_labels(settings.LABELS_FILE) Logger.log_field("Labels Loaded", len(loader.label_map)) # Load the samples from the set that we want. samples = Loader.load_sample_set(set_index) loaded_samples = [ s for s in samples if (s.is_locally_loaded and len(s.detect_regions) > 0) ] # How many samples loaded? n_loaded_samples = len(loaded_samples) Logger.log_field("Samples with Images", n_loaded_samples) if n_loaded_samples == 0: raise Exception( "None of the samples in this set have been downloaded! " "Please run cmd_load_sample_images first for this set.")
""" from __future__ import print_function from __future__ import division from __future__ import absolute_import import argparse import os from modules.ai.single_cell_classifier.scc_trainer import SCCTrainer from tools.util.logger import Logger __author__ = "Jakrin Juangbhanich" __email__ = "*****@*****.**" __version__ = "0.0.1" parser = argparse.ArgumentParser(description='<Script Info>') parser.add_argument('-i', '--input', default='data', type=str, help="<help>") parser.add_argument('-f', '--flag', action="store_true", help="<help>") args = parser.parse_args() if __name__ == "__main__": input_path = os.path.join("input", args.input) Logger.log_header("Running Cell-Scan Trainer", with_gap=True) Logger.log_field("Version", __version__) Logger.log_field("Input Folder", input_path) scc_trainer = SCCTrainer() scc_trainer.process(input_path)
args = get_args() set_index = args.set_index max_threads = args.max_threads if __name__ == "__main__": # Load the project settings and required modules. Logger.log_special("Running Sample Loader", with_gap=True) settings = ProjectSettings("settings.yaml") set_path = os.path.join(settings.SAMPLES_DIRECTORY, f"sample_set_{set_index}.json") if not os.path.exists(set_path): Logger.log_field( "Error", "No file found at {}. Have you created the samples using cmd_create_samples yet?" ) exit(1) Logger.log_special("Begin Sample Image Download", with_gap=True) samples = Loader.load_sample_set_from_file(set_path) unloaded_samples = [s for s in samples if not s.is_locally_loaded] n_unloaded_samples = len(unloaded_samples) n_samples = len(samples) Logger.log_field("Samples Loaded", "{}/{}".format(n_samples - n_unloaded_samples, n_samples)) i = 0 for sample in unloaded_samples: while True: