def run(self):
model_path = self.inf_model_path
MODEL_MAKER = Model_NP_XY if self.model_mode == 'np+xy' else Model_NP_DIST
pred_config = PredictConfig(
model=MODEL_MAKER(),
session_init=get_model_loader(model_path),
input_names=self.eval_inf_input_tensor_names,
output_names=self.eval_inf_output_tensor_names)
predictor = OfflinePredictor(pred_config)
for norm_target in self.inf_norm_codes:
norm_dir = '%s/%s/' % (self.inf_norm_root_dir, norm_target)
norm_save_dir = '%s/%s/' % (self.inf_output_dir, norm_target)
# TODO: cache list to check later norm dir has same number of files
file_list = glob.glob('%s/*%s' % (norm_dir, self.inf_imgs_ext))
file_list.sort() # ensure same order
rm_n_mkdir(norm_save_dir)
for filename in file_list:
filename = os.path.basename(filename)
basename = filename.split('.')[0]
print(basename, norm_target, end=' ', flush=True)
##
img = cv2.imread(norm_dir + filename)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
##
pred_map = self.__gen_prediction(img, predictor)
sio.savemat('%s/%s.mat' % (norm_save_dir, basename),
{'result': [pred_map]})
print('FINISH')
def run(self):
predictor = OfflinePredictor(self.gen_pred_config())
for num, data_dir in enumerate(self.inf_data_list):
save_dir = os.path.join(self.inf_output_dir, str(num))
print(save_dir)
file_list = glob.glob(
os.path.join(data_dir, "*{}".format(self.inf_imgs_ext)))
file_list.sort() # ensure same order
rm_n_mkdir(save_dir)
for filename in file_list:
filename = os.path.basename(filename)
basename = filename.split(".")[0]
print(data_dir, basename, end=" ", flush=True)
##
img = cv2.imread(os.path.join(data_dir, filename))
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
##
pred_map = self.__gen_prediction(img, predictor)
sio.savemat(
os.path.join(save_dir, "{}.mat".format(basename)),
{"result": [pred_map]},
)
print(f"Finished. {datetime.now().strftime('%H:%M:%S.%f')}")
def stain_normilize(img_dir, save_dir, stain_norm_target, norm_brightness=False):
file_list = glob.glob(os.path.join(img_dir, '*.png'))
file_list.sort()
if norm_brightness:
standardizer = staintools.LuminosityStandardizer()
stain_normalizer = staintools.StainNormalizer(method='vahadane')
# dict of paths to target image and dir code to make output folder
# {'/data/TCGA-21-5784-01Z-00-DX1.tif' : '5784'}
# stain_norm_targets = {k : v for k, v in zip(glob.glob(os.path.join(targets_dir, '*.*')), range(len(glob.glob(os.path.join(targets_dir, '*.*')))))}
# stain_norm_target = {target : '1'}
target_img = cv2.imread(stain_norm_target)
target_img = cv2.cvtColor(target_img, cv2.COLOR_BGR2RGB)
if norm_brightness:
target_img = standardizer.standardize(target_img)
stain_normalizer.fit(target_img)
norm_dir = save_dir
rm_n_mkdir(norm_dir)
for img_path in file_list:
filename = os.path.basename(img_path)
basename = filename.split('.')[0]
img = cv2.imread(img_path)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
if norm_brightness:
img = standardizer.standardize(img)
img = stain_normalizer.transform(img)
img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
cv2.imwrite(os.path.join(norm_dir, '{}.png'.format(basename)), img)
print(f"Saved {os.path.join(norm_dir, '{}.png'.format(basename))}.")
def __infer(self, inf_model_path, input_file_list, pred_out_dir):
augmentors = self.infer_augmentors()
saved_state_dict = torch.load(inf_model_path)
net_g = netdesc.Generator(1, 3).to('cuda')
net_g.load_state_dict(saved_state_dict['net_g'])
net_g = torch.nn.DataParallel(net_g).to('cuda')
filepath_list = input_file_list
basename_list = [os.path.basename(path).split('.')[0] for path in input_file_list]
input_info = list(zip(filepath_list, basename_list))
infer_dataset = SerialLoader(
input_info, run_mode='infer',
shape_augs=iaa.Sequential(augmentors[0]),
input_augs=iaa.Sequential(augmentors[1]))
dataloader = data.DataLoader(infer_dataset,
num_workers=self.nr_procs_infer,
batch_size=1,
drop_last=False)
utils.rm_n_mkdir(pred_out_dir)
self.__run_list(net_g, dataloader, pred_out_dir)
return
def run(self):
exporter = ModelExporter(self.gen_pred_config())
rm_n_mkdir(self.model_export_dir)
exporter.export_compact(
filename="{}/compact.pb".format(self.model_export_dir))
exporter.export_serving(
os.path.join(self.model_export_dir, "serving"),
signature_name="serving_default",
)
print(f"Saved model to {self.model_export_dir}.")
def run(self, save_only):
if self.inf_auto_find_chkpt:
self.inf_model_path = os.path.join(self.save_dir, str(max([int(x) for x in [name for name in os.listdir(self.save_dir) if os.path.isdir(os.path.join(self.save_dir, name))]])))
print(f"Inference model path: <{self.inf_model_path}>")
print('-----Auto Selecting Checkpoint Basing On "%s" Through "%s" Comparison' % \
(self.inf_auto_metric, self.inf_auto_comparator))
model_path, stat = get_best_chkpts(self.inf_model_path, self.inf_auto_metric, self.inf_auto_comparator)
print('Selecting: %s' % model_path)
print('Having Following Statistics:')
for key, value in stat.items():
print('\t%s: %s' % (key, value))
else:
model_path = self.inf_model_path
model_constructor = self.get_model()
pred_config = PredictConfig(
model = model_constructor(),
session_init = get_model_loader(model_path),
input_names = self.eval_inf_input_tensor_names,
output_names = self.eval_inf_output_tensor_names)
predictor = OfflinePredictor(pred_config)
if save_only:
exporter = ModelExporter(pred_config)
rm_n_mkdir(self.model_export_dir)
print ('{}/compact.pb'.format(self.model_export_dir))
exporter.export_compact(filename='{}/compact.pb'.format(self.model_export_dir))
exporter.export_serving(os.path.join(self.model_export_dir, 'serving'), signature_name='serving_default')
return
for num, data_dir in enumerate(self.inf_data_list):
save_dir = os.path.join(self.inf_output_dir, str(num))
file_list = glob.glob(os.path.join(data_dir, '*{}'.format(self.inf_imgs_ext)))
file_list.sort() # ensure same order
rm_n_mkdir(save_dir)
for filename in file_list:
filename = os.path.basename(filename)
basename = filename.split('.')[0]
print(data_dir, basename, end=' ', flush=True)
##
img = cv2.imread(os.path.join(data_dir, filename))
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
##
pred_map = self.__gen_prediction(img, predictor)
sio.savemat(os.path.join(save_dir,'{}.mat'.format(basename)), {'result':[pred_map]})
print(f"Finished. {datetime.now().strftime('%H:%M:%S.%f')}")
def run(self):
if self.inf_auto_find_chkpt:
print('-----Auto Selecting Checkpoint Basing On "%s" Through "%s" Comparison' % \
(self.inf_auto_metric, self.inf_auto_comparator))
model_path, stat = get_best_chkpts(self.save_dir,
self.inf_auto_metric,
self.inf_auto_comparator)
print('Selecting: %s' % model_path)
print('Having Following Statistics:')
for key, value in stat.items():
print('\t%s: %s' % (key, value))
else:
model_path = self.inf_model_path
model_constructor = self.get_model()
pred_config = PredictConfig(
model=model_constructor(),
session_init=get_model_loader(model_path),
input_names=self.eval_inf_input_tensor_names,
output_names=self.eval_inf_output_tensor_names)
predictor = OfflinePredictor(pred_config)
for data_dir_set in self.inf_data_list:
data_root_dir = data_dir_set[0]
data_out_code = data_dir_set[1]
for subdir in data_dir_set[2:]:
data_dir = '%s/%s/' % (data_root_dir, subdir)
save_dir = '%s/%s/%s' % (self.inf_output_dir, data_out_code,
subdir)
file_list = glob.glob('%s/*%s' % (data_dir, self.inf_imgs_ext))
file_list.sort() # ensure same order
rm_n_mkdir(save_dir)
for filename in file_list:
filename = os.path.basename(filename)
basename = filename.split('.')[0]
print(data_dir, basename, end=' ', flush=True)
##
img = cv2.imread(data_dir + filename)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
##
pred_map = self.__gen_prediction(img, predictor)
sio.savemat('%s/%s.mat' % (save_dir, basename),
{'result': [pred_map]})
print('FINISH')
def run(self):
if self.inf_auto_find_chkpt:
print(
'-----Auto Selecting Checkpoint Basing On "%s" Through "%s" Comparison'
% (self.inf_auto_metric, self.inf_auto_comparator))
model_path, stat = get_best_chkpts(self.save_dir,
self.inf_auto_metric,
self.inf_auto_comparator)
print("Selecting: %s" % model_path)
print("Having Following Statistics:")
for key, value in stat.items():
print("\t%s: %s" % (key, value))
else:
model_path = self.inf_model_path
model_constructor = self.get_model()
pred_config = PredictConfig(
model=model_constructor(),
session_init=get_model_loader(model_path),
input_names=self.eval_inf_input_tensor_names,
output_names=self.eval_inf_output_tensor_names,
create_graph=False,
)
predictor = OfflinePredictor(pred_config)
for data_dir in self.inf_data_list:
save_dir = self.inf_output_dir + "/raw/"
file_list = glob.glob("%s/*%s" % (data_dir, self.inf_imgs_ext))
file_list.sort() # ensure same order
rm_n_mkdir(save_dir)
for filename in file_list:
start = time.time()
filename = os.path.basename(filename)
basename = filename.split(".")[0]
print(data_dir, basename, end=" ", flush=True)
##
img = cv2.imread(data_dir + filename)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
pred_map = self.__gen_prediction(img, predictor)
np.save("%s/%s.npy" % (save_dir, basename), [pred_map])
end = time.time()
diff = str(round(end - start, 2))
print("FINISH. TIME: %s" % diff)
def process_wsi_list(self, run_args):
"""Process a list of whole-slide images.
Args:
run_args: arguments as defined in run_infer.py
"""
self._parse_args(run_args)
if not os.path.exists(self.cache_path):
rm_n_mkdir(self.cache_path)
if not os.path.exists(self.output_dir + "/json/"):
rm_n_mkdir(self.output_dir + "/json/")
if self.save_thumb:
if not os.path.exists(self.output_dir + "/thumb/"):
rm_n_mkdir(self.output_dir + "/thumb/")
if self.save_mask:
if not os.path.exists(self.output_dir + "/mask/"):
rm_n_mkdir(self.output_dir + "/mask/")
wsi_path_list = glob.glob(self.input_dir + "/*")
wsi_path_list.sort() # ensure ordering
for wsi_path in wsi_path_list[:]:
wsi_base_name = pathlib.Path(wsi_path).stem
msk_path = "%s/%s.png" % (self.input_mask_dir, wsi_base_name)
if self.save_thumb or self.save_mask:
output_file = "%s/json/%s.json" % (self.output_dir,
wsi_base_name)
else:
output_file = "%s/%s.json" % (self.output_dir, wsi_base_name)
if os.path.exists(output_file):
log_info("Skip: %s" % wsi_base_name)
continue
try:
log_info("Process: %s" % wsi_base_name)
self.process_single_file(wsi_path, msk_path, self.output_dir)
log_info("Finish")
except:
logging.exception("Crash")
rm_n_mkdir(self.cache_path) # clean up all cache
return
def run_once(self,
opt,
run_engine_opt,
log_dir,
prev_log_dir=None,
fold_idx=0):
"""Simply run the defined run_step of the related method once."""
check_manual_seed(self.seed)
log_info = {}
if self.logging:
# check_log_dir(log_dir)
rm_n_mkdir(log_dir)
tfwriter = SummaryWriter(log_dir=log_dir)
json_log_file = log_dir + "/stats.json"
with open(json_log_file, "w") as json_file:
json.dump({}, json_file) # create empty file
log_info = {
"json_file": json_log_file,
"tfwriter": tfwriter,
}
####
loader_dict = {}
for runner_name, runner_opt in run_engine_opt.items():
loader_dict[runner_name] = self._get_datagen(
opt["batch_size"][runner_name],
runner_name,
opt["target_info"]["gen"],
nr_procs=runner_opt["nr_procs"],
fold_idx=fold_idx,
)
####
def get_last_chkpt_path(prev_phase_dir, net_name):
stat_file_path = prev_phase_dir + "/stats.json"
with open(stat_file_path) as stat_file:
info = json.load(stat_file)
epoch_list = [int(v) for v in info.keys()]
last_chkpts_path = "%s/%s_epoch=%d.tar" % (
prev_phase_dir,
net_name,
max(epoch_list),
)
return last_chkpts_path
# TODO: adding way to load pretrained weight or resume the training
# parsing the network and optimizer information
net_run_info = {}
net_info_opt = opt["run_info"]
for net_name, net_info in net_info_opt.items():
assert inspect.isclass(net_info["desc"]) or inspect.isfunction(
net_info["desc"]
), "`desc` must be a Class or Function which instantiate NEW objects !!!"
net_desc = net_info["desc"]()
# TODO: customize print-out for each run ?
# summary_string(net_desc, (3, 270, 270), device='cpu')
pretrained_path = net_info["pretrained"]
if pretrained_path is not None:
if pretrained_path == -1:
# * depend on logging format so may be broken if logging format has been changed
pretrained_path = get_last_chkpt_path(
prev_log_dir, net_name)
net_state_dict = torch.load(pretrained_path)["desc"]
else:
chkpt_ext = os.path.basename(pretrained_path).split(
".")[-1]
if chkpt_ext == "npz":
net_state_dict = dict(np.load(pretrained_path))
net_state_dict = {
k: torch.from_numpy(v)
for k, v in net_state_dict.items()
}
elif chkpt_ext == "tar": # ! assume same saving format we desire
net_state_dict = torch.load(pretrained_path)["desc"]
colored_word = colored(net_name, color="red", attrs=["bold"])
print("Model `%s` pretrained path: %s" %
(colored_word, pretrained_path))
# load_state_dict returns (missing keys, unexpected keys)
net_state_dict = convert_pytorch_checkpoint(net_state_dict)
load_feedback = net_desc.load_state_dict(net_state_dict,
strict=False)
# * uncomment for your convenience
print("Missing Variables: \n", load_feedback[0])
print("Detected Unknown Variables: \n", load_feedback[1])
# * extremely slow to pass this on DGX with 1 GPU, why (?)
net_desc = DataParallel(net_desc)
net_desc = net_desc.to("cuda")
# print(net_desc) # * dump network definition or not?
optimizer, optimizer_args = net_info["optimizer"]
optimizer = optimizer(net_desc.parameters(), **optimizer_args)
# TODO: expand for external aug for scheduler
nr_iter = opt["nr_epochs"] * len(loader_dict["train"])
scheduler = net_info["lr_scheduler"](optimizer)
net_run_info[net_name] = {
"desc": net_desc,
"optimizer": optimizer,
"lr_scheduler": scheduler,
# TODO: standardize API for external hooks
"extra_info": net_info["extra_info"],
}
# parsing the running engine configuration
assert ("train" in run_engine_opt
), "No engine for training detected in description file"
# initialize runner and attach callback afterward
# * all engine shared the same network info declaration
runner_dict = {}
for runner_name, runner_opt in run_engine_opt.items():
runner_dict[runner_name] = RunEngine(
dataloader=loader_dict[runner_name],
engine_name=runner_name,
run_step=runner_opt["run_step"],
run_info=net_run_info,
log_info=log_info,
)
for runner_name, runner in runner_dict.items():
callback_info = run_engine_opt[runner_name]["callbacks"]
for event, callback_list, in callback_info.items():
for callback in callback_list:
if callback.engine_trigger:
triggered_runner_name = callback.triggered_engine_name
callback.triggered_engine = runner_dict[
triggered_runner_name]
runner.add_event_handler(event, callback)
# retrieve main runner
main_runner = runner_dict["train"]
main_runner.state.logging = self.logging
main_runner.state.log_dir = log_dir
# start the run loop
main_runner.run(opt["nr_epochs"])
print("\n")
print("########################################################")
print("########################################################")
print("\n")
return
img_dirs = cfg.out_preproc if normalized else cfg.img_dirs
print(f"Using folders <{list(img_dirs.values())}> as input")
print(f"Saving results to <{list(cfg.out_extract.values())}>")
for data_mode in img_dirs.keys():
xtractor = PatchExtractor(cfg.win_size, cfg.step_size)
img_dir = img_dirs[data_mode]
ann_dir = cfg.labels_dirs[data_mode]
file_list = glob.glob(os.path.join(img_dir, '*{}'.format(cfg.img_ext)))
file_list.sort()
out_dir = cfg.out_extract[data_mode]
rm_n_mkdir(out_dir)
for filename in file_list:
filename = os.path.basename(filename)
basename = filename.split('.')[0]
print('Mode: {}, filename - {}'.format(data_mode, filename))
img = cv2.imread(os.path.join(img_dir, '{}{}'.format(basename, cfg.img_ext)))
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
if cfg.type_classification:
# # assumes that ann is HxWx2 (nuclei class labels are available at index 1 of C)
ann = np.load(os.path.join(ann_dir, '{}.npy'.format(basename)), allow_pickle=True)
# ann_inst = ann[...,0]
# ann_type = ann[...,1]
ann_inst = ann.item().get('inst_map')
ann_type = ann.item().get('type_map')
def process_file_list(self, run_args):
"""
Process a single image tile < 5000x5000 in size.
"""
for variable, value in run_args.items():
self.__setattr__(variable, value)
# * depend on the number of samples and their size, this may be less efficient
patterning = lambda x: re.sub("([\[\]])", "[\\1]", x)
file_path_list = glob.glob(patterning("%s/*" % self.input_dir))
file_path_list.sort() # ensure same order
assert len(file_path_list) > 0, 'Not Detected Any Files From Path'
rm_n_mkdir(self.output_dir + '/json/')
rm_n_mkdir(self.output_dir + '/mat/')
rm_n_mkdir(self.output_dir + '/overlay/')
if self.save_qupath:
rm_n_mkdir(self.output_dir + "/qupath/")
def proc_callback(results):
"""Post processing callback.
Output format is implicit assumption, taken from `_post_process_patches`
"""
img_name, pred_map, pred_inst, inst_info_dict, overlaid_img = results
inst_type = [[k, v["type"]] for k, v in inst_info_dict.items()]
inst_type = np.array(inst_type)
mat_dict = {
"inst_map": pred_inst,
"inst_type": inst_type,
}
if self.nr_types is None: # matlab does not have None type array
mat_dict.pop("inst_type", None)
if self.save_raw_map:
mat_dict["raw_map"] = pred_map
save_path = "%s/mat/%s.mat" % (self.output_dir, img_name)
sio.savemat(save_path, mat_dict)
save_path = "%s/overlay/%s.png" % (self.output_dir, img_name)
cv2.imwrite(save_path, cv2.cvtColor(overlaid_img,
cv2.COLOR_RGB2BGR))
if self.save_qupath:
nuc_val_list = list(inst_info_dict.values())
nuc_type_list = np.array([v["type"] for v in nuc_val_list])
nuc_coms_list = np.array([v["centroid"] for v in nuc_val_list])
save_path = "%s/qupath/%s.tsv" % (self.output_dir, img_name)
convert_format.to_qupath(save_path, nuc_coms_list,
nuc_type_list, self.type_info_dict)
save_path = "%s/json/%s.json" % (self.output_dir, img_name)
self.__save_json(save_path, inst_info_dict, None)
return img_name
def detach_items_of_uid(items_list, uid, nr_expected_items):
item_counter = 0
detached_items_list = []
remained_items_list = []
while True:
pinfo, pdata = items_list.pop(0)
pinfo = np.squeeze(pinfo)
if pinfo[-1] == uid:
detached_items_list.append([pinfo, pdata])
item_counter += 1
else:
remained_items_list.append([pinfo, pdata])
if item_counter == nr_expected_items:
break
# do this to ensure the ordering
remained_items_list = remained_items_list + items_list
return detached_items_list, remained_items_list
proc_pool = None
future_list = []
if self.nr_post_proc_workers > 0:
proc_pool = ProcessPoolExecutor(self.nr_post_proc_workers)
while len(file_path_list) > 0:
hardware_stats = psutil.virtual_memory()
available_ram = getattr(hardware_stats, "available")
available_ram = int(available_ram * 0.6)
# available_ram >> 20 for MB, >> 30 for GB
# TODO: this portion looks clunky but seems hard to detach into separate func
# * caching N-files into memory such that their expected (total) memory usage
# * does not exceed the designated percentage of currently available memory
# * the expected memory is a factor w.r.t original input file size and
# * must be manually provided
file_idx = 0
use_path_list = []
cache_image_list = []
cache_patch_info_list = []
cache_image_info_list = []
while len(file_path_list) > 0:
file_path = file_path_list.pop(0)
img = cv2.imread(file_path)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
src_shape = img.shape
img, patch_info, top_corner = _prepare_patching(
img, self.patch_input_shape, self.patch_output_shape, True)
self_idx = np.full(patch_info.shape[0],
file_idx,
dtype=np.int32)
patch_info = np.concatenate([patch_info, self_idx[:, None]],
axis=-1)
# ? may be expensive op
patch_info = np.split(patch_info, patch_info.shape[0], axis=0)
patch_info = [np.squeeze(p) for p in patch_info]
# * this factor=5 is only applicable for HoVerNet
expected_usage = sys.getsizeof(img) * 5
available_ram -= expected_usage
if available_ram < 0:
break
file_idx += 1
# if file_idx == 4: break
use_path_list.append(file_path)
cache_image_list.append(img)
cache_patch_info_list.extend(patch_info)
# TODO: refactor to explicit protocol
cache_image_info_list.append(
[src_shape, len(patch_info), top_corner])
# * apply neural net on cached data
dataset = SerializeFileList(cache_image_list,
cache_patch_info_list,
self.patch_input_shape)
dataloader = data.DataLoader(
dataset,
num_workers=self.nr_inference_workers,
batch_size=self.batch_size,
drop_last=False,
)
pbar = tqdm.tqdm(
desc="Process Patches",
leave=True,
total=int(len(cache_patch_info_list) / self.batch_size) + 1,
ncols=80,
ascii=True,
position=0,
)
accumulated_patch_output = []
for batch_idx, batch_data in enumerate(dataloader):
sample_data_list, sample_info_list = batch_data
sample_output_list = self.run_step(sample_data_list)
sample_info_list = sample_info_list.numpy()
curr_batch_size = sample_output_list.shape[0]
sample_output_list = np.split(sample_output_list,
curr_batch_size,
axis=0)
sample_info_list = np.split(sample_info_list,
curr_batch_size,
axis=0)
sample_output_list = list(
zip(sample_info_list, sample_output_list))
accumulated_patch_output.extend(sample_output_list)
pbar.update()
pbar.close()
# * parallely assemble the processed cache data for each file if possible
for file_idx, file_path in enumerate(use_path_list):
image_info = cache_image_info_list[file_idx]
file_ouput_data, accumulated_patch_output = detach_items_of_uid(
accumulated_patch_output, file_idx, image_info[1])
# * detach this into func and multiproc dispatch it
src_pos = image_info[
2] # src top left corner within padded image
src_image = cache_image_list[file_idx]
src_image = src_image[src_pos[0]:src_pos[0] + image_info[0][0],
src_pos[1]:src_pos[1] +
image_info[0][1], ]
base_name = pathlib.Path(file_path).stem
file_info = {
"src_shape": image_info[0],
"src_image": src_image,
"name": base_name,
}
post_proc_kwargs = {
"nr_types": self.nr_types,
"return_centroids": True,
} # dynamicalize this
overlay_kwargs = {
"draw_dot": self.draw_dot,
"type_colour": self.type_info_dict,
"line_thickness": 1,
}
func_args = (
self.post_proc_func,
post_proc_kwargs,
file_ouput_data,
file_info,
overlay_kwargs,
)
# dispatch for parallel post-processing
if proc_pool is not None:
proc_future = proc_pool.submit(_post_process_patches,
*func_args)
# ! manually poll future and call callback later as there is no guarantee
# ! that the callback is called from main thread
future_list.append(proc_future)
else:
proc_output = _post_process_patches(*func_args)
proc_callback(proc_output)
if proc_pool is not None:
# loop over all to check state a.k.a polling
for future in as_completed(future_list):
# TODO: way to retrieve which file crashed ?
# ! silent crash, cancel all and raise error
if future.exception() is not None:
log_info("Silent Crash")
# ! cancel somehow leads to cascade error later
# ! so just poll it then crash once all future
# ! acquired for now
# for future in future_list:
# future.cancel()
# break
else:
file_path = proc_callback(future.result())
log_info("Done Assembling %s" % file_path)
return
def run(self):
if self.inf_auto_find_chkpt:
print(
'-----Auto Selecting Checkpoint Basing On "%s" Through "%s" Comparison'
% (self.inf_auto_metric, self.inf_auto_comparator))
model_path, stat = get_best_chkpts(self.save_dir,
self.inf_auto_metric,
self.inf_auto_comparator)
print('Selecting: %s' % model_path)
print('Having Following Statistics:')
for key, value in stat.items():
print('\t%s: %s' % (key, value))
else:
model_path = self.inf_model_path
####
save_dir = self.inf_output_dir
predict_list = [['case', 'prediction']]
file_load_img = HDF5Matrix(
self.inf_data_list[0] + 'camelyonpatch_level_2_split_test_x.h5',
'x')
file_load_lab = HDF5Matrix(
self.inf_data_list[0] + 'camelyonpatch_level_2_split_test_y.h5',
'y')
true_list = []
prob_list = []
pred_list = []
num_ims = file_load_img.shape[0]
last_step = math.floor(num_ims / self.inf_batch_size)
last_step = self.inf_batch_size * last_step
last_batch = num_ims - last_step
count = 0
for start_batch in range(0, last_step + 1, self.inf_batch_size):
sys.stdout.write("\rProcessed (%d/%d)" % (start_batch, num_ims))
sys.stdout.flush()
if start_batch != last_step:
img = file_load_img[start_batch:start_batch +
self.inf_batch_size]
img = img.astype('uint8')
lab = np.squeeze(file_load_lab[start_batch:start_batch +
self.inf_batch_size])
else:
img = file_load_img[start_batch:start_batch + last_batch]
img = img.astype('uint8')
lab = np.squeeze(file_load_lab[start_batch:start_batch +
last_batch])
prob, pred = self.__gen_prediction(img, predictor)
for j in range(prob.shape[0]):
predict_list.append([str(count), str(prob[j])])
count += 1
prob_list.extend(prob)
pred_list.extend(pred)
true_list.extend(lab)
prob_list = np.array(prob_list)
pred_list = np.array(pred_list)
true_list = np.array(true_list)
accuracy = (pred_list == true_list).sum() / np.size(true_list)
error = (pred_list != true_list).sum() / np.size(true_list)
print('Accurcy (%): ', 100 * accuracy)
print('Error (%): ', 100 * error)
if self.model_mode == 'class_pcam':
auc = roc_auc_score(true_list, prob_list)
print('AUC: ', auc)
# Save predictions to csv
rm_n_mkdir(save_dir)
for result in predict_list:
predict_file = open('%s/predict.csv' % save_dir, "a")
predict_file.write(result[0])
predict_file.write(',')
predict_file.write(result[1])
predict_file.write("\n")
predict_file.close()
import numpy as np
import os
import glob
import scipy.io as sio
from misc.utils import get_inst_centroid, rm_n_mkdir
from metrics.stats_utils import remap_label
ann_dir = '/home/test/GhulamMurtaza/panNuke/Test/Labels/' # * directory contains .npy
filepath_list = glob.glob('%s/*.npy' % ann_dir)
save_dir = 'GroundTruth/dump/' # directory to save summarized info about nuclei
rm_n_mkdir(save_dir)
for path in filepath_list:
basename = os.path.basename(path).split('.')[0]
true_map = np.load(path)
true_inst = true_map[..., 0]
true_type = true_map[..., 1]
true_inst = remap_label(true_inst, by_size=True)
true_inst_centroid = get_inst_centroid(true_inst)
#### * Get class of each instance id, stored at index id-1
# for ground truth instance blob
true_id_list = list(np.unique(true_inst))[1:] # exclude background
true_inst_type = np.full(len(true_id_list), -1, dtype=np.int32)
for idx, inst_id in enumerate(true_id_list):
inst_type = true_type[true_inst == inst_id]
type_list, type_pixels = np.unique(inst_type, return_counts=True)
inst_type = type_list[np.argmax(type_pixels)]
if inst_type != 0: # there are artifact nuclei (background types)
def run(self):
if self.inf_auto_find_chkpt:
print(
'-----Auto Selecting Checkpoint Basing On "%s" Through "%s" Comparison'
% (self.inf_auto_metric, self.inf_auto_comparator))
model_path, stat = get_best_chkpts(self.save_dir,
self.inf_auto_metric,
self.inf_auto_comparator)
print("Selecting: %s" % model_path)
print("Having Following Statistics:")
for key, value in stat.items():
print("\t%s: %s" % (key, value))
else:
model_path = self.inf_model_path
model_constructor = self.get_model()
pred_config = PredictConfig(
model=model_constructor(),
session_init=get_model_loader(model_path),
input_names=self.eval_inf_input_tensor_names,
output_names=self.eval_inf_output_tensor_names,
create_graph=False)
predictor = OfflinePredictor(pred_config)
####
save_dir = self.inf_output_dir
predict_list = [["case", "prediction"]]
file_load_img = HDF5Matrix(
self.inf_data_list[0] + "camelyonpatch_level_2_split_test_x.h5",
"x")
file_load_lab = HDF5Matrix(
self.inf_data_list[0] + "camelyonpatch_level_2_split_test_y.h5",
"y")
true_list = []
prob_list = []
pred_list = []
num_ims = file_load_img.shape[0]
last_step = math.floor(num_ims / self.inf_batch_size)
last_step = self.inf_batch_size * last_step
last_batch = num_ims - last_step
count = 0
for start_batch in range(0, last_step + 1, self.inf_batch_size):
sys.stdout.write("\rProcessed (%d/%d)" % (start_batch, num_ims))
sys.stdout.flush()
if start_batch != last_step:
img = file_load_img[start_batch:start_batch +
self.inf_batch_size]
img = img.astype("uint8")
lab = np.squeeze(file_load_lab[start_batch:start_batch +
self.inf_batch_size])
else:
img = file_load_img[start_batch:start_batch + last_batch]
img = img.astype("uint8")
lab = np.squeeze(file_load_lab[start_batch:start_batch +
last_batch])
prob, pred = self.__gen_prediction(img, predictor)
for j in range(prob.shape[0]):
predict_list.append([str(count), str(prob[j])])
count += 1
prob_list.extend(prob)
pred_list.extend(pred)
true_list.extend(lab)
prob_list = np.array(prob_list)
pred_list = np.array(pred_list)
true_list = np.array(true_list)
accuracy = (pred_list == true_list).sum() / np.size(true_list)
error = (pred_list != true_list).sum() / np.size(true_list)
print("Accurcy (%): ", 100 * accuracy)
print("Error (%): ", 100 * error)
if self.model_mode == "class_pcam":
auc = roc_auc_score(true_list, prob_list)
print("AUC: ", auc)
# Save predictions to csv
rm_n_mkdir(save_dir)
for result in predict_list:
predict_file = open("%s/predict.csv" % save_dir, "a")
predict_file.write(result[0])
predict_file.write(",")
predict_file.write(result[1])
predict_file.write("\n")
predict_file.close()
stain_normalizer = staintools.StainNormalizer(method='vahadane')
# dict of paths to target image and dir code to make output folder
stain_norm_target = {
'../data/TCGA-21-5784-01Z-00-DX1.tif' : '5784'
}
for target_path, target_code in stain_norm_target.items():
target_img = cv2.imread(target_path)
target_img = cv2.cvtColor(target_img, cv2.COLOR_BGR2RGB)
if norm_brightness:
target_img = standardizer.transform(target_img)
stain_normalizer.fit(target_img)
norm_dir = "%s/%s/" % (save_dir, target_code)
rm_n_mkdir(norm_dir)
for img_path in file_list:
filename = os.path.basename(img_path)
basename = filename.split('.')[0]
print(basename)
img = cv2.imread(img_path)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
if norm_brightness:
img = standardizer.transform(img)
img = stain_normalizer.transform(img)
img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
cv2.imwrite("%s/%s.png" % (norm_dir, basename), img)
