def main(input_image):
# *************************************************************************
with tf.gfile.Open(input_image, 'rb') as f:
try:
image = Image.open(f).convert('RGB')
except (tf.errors.OutOfRangeError, OSError) as e:
print 'Exception!'
# *************************************************************************
config = get_config(CONFIG)
config.model.rcnn.proposals.total_max_detections = MAX_DET
config.model.rcnn.proposals.min_prob_threshold = MIN_PROB
network = PredictorNetwork(config)
objects = network.predict_image(image)
print '************************* Num of Objects : ', len(objects)
# *************************************************************************
ref_image = cv2.cvtColor(np.array(image), cv2.COLOR_BGR2RGB)
pref_list, images, ids = classify_cellscropping.load_json(ref_image,
preds=objects)
print '************************* Num of Cropped Objects : ', np.shape(ids)
# *************************************************************************
final_json = classify_main.predict(images, ids, CKPT)
output_json_path = os.path.join(
OUTPUT_JSON_DIR, '%s.json' % input_image.split('/')[-1][:-4])
with open(output_json_path, 'w') as fp:
json.dump(final_json, fp)
def __init__(self, checkpoint=None, config=None, prob=0.7, classes=None):
"""Instantiate a detector object with the appropriate config.
Arguments:
checkpoint (str): Checkpoint id or alias to instantiate the
detector as.
config (dict): Configuration parameters describing the desired
model. See `get_config` to load a config file.
Note:
Only one of the parameters must be specified. If none is, we
default to loading the checkpoint indicated by
`DEFAULT_CHECKPOINT`.
"""
if checkpoint is not None and config is not None:
raise ValueError(
"Only one of `checkpoint` or `config` must be specified in "
"order to instantiate a Detector.")
if checkpoint is None and config is None:
# Neither checkpoint no config specified, default to
# `DEFAULT_CHECKPOINT`.
checkpoint = self.DEFAULT_CHECKPOINT
if checkpoint:
config = get_checkpoint_config(checkpoint)
# Prevent the model itself from filtering its proposals (default
# value of 0.5 is in use in the configs).
# TODO: A model should always return all of its predictions. The
# filtering should be done (if at all) by PredictorNetwork.
if config.model.type == "fasterrcnn":
config.model.rcnn.proposals.min_prob_threshold = 0.0
elif config.model.type == "ssd":
config.model.proposals.min_prob_threshold = 0.0
# TODO: Remove dependency on `PredictorNetwork` or clearly separate
# responsibilities.
self._network = PredictorNetwork(config)
self.prob = prob
# Use the labels when available, integers when not.
self._model_classes = (self._network.class_labels
if self._network.class_labels else list(
range(config.model.network.num_classes)))
if classes:
self.classes = set(classes)
if not set(self._model_classes).issuperset(self.classes):
raise ValueError(
"`classes` must be contained in the detector's classes. "
"Available classes are: {}.".format(self._model_classes))
else:
self.classes = set(self._model_classes)
def inference(self, model, input_dir, score_threshold):
config = get_checkpoint_config(network, prompt=False)
if not config:
return flask.jsonify({'error': 'network not found'})
try:
predictor = PredictorNetwork(config)
paths = os.listdir(input_dir) if os.path.isdir(input_dir) else [
input_dir
]
for image_file in paths:
image = Image.open(image_file).conver('RGB')
predictor.predict_image(image)
except Exception as e:
# An error occurred loading the model; interrupt the whole server.
_thread.interrupt_main()
def start_network(config):
global PREDICTOR_NETWORK
try:
PREDICTOR_NETWORK = PredictorNetwork(config)
except Exception as e:
# An error occurred loading the model; interrupt the whole server.
tf.logging.error(e)
_thread.interrupt_main()
def main(input_image):
tf.reset_default_graph()
# **********************************************************************
with tf.gfile.Open(input_image, 'rb') as f:
try:
image = Image.open(f).convert('RGB')
except (tf.errors.OutOfRangeError, OSError) as e:
print 'Exception!'
# **********************************************************************
config = get_config(CONFIG)
config.model.rcnn.proposals.total_max_detections = MAX_DET
config.model.rcnn.proposals.min_prob_threshold = MIN_PROB
network = PredictorNetwork(config)
objects = network.predict_image(image)
print '************************************** Num of Objects : ', len(
objects)
# **********************************************************************
ref_image = cv2.cvtColor(np.array(image), cv2.COLOR_BGR2RGB)
pref_list, images, ids = cell_croppings.load_json(ref_image, preds=objects)
print '************************************** Num of Cropped Objects : ', np.shape(
ids)
# **********************************************************************
if not os.path.exists(CROP_CELLS_DIR):
os.makedirs(CROP_CELLS_DIR)
for i in range(ids.shape[0]):
# SRC_DIR / IMAGE_ID _ COORDS . LABEL
dst_filename = '%s_%s_%s.png' % (os.path.basename(input_image)[:-4],
ids[i], i)
dst_img_name = os.path.join(CROP_CELLS_DIR, dst_filename)
cv2.imwrite(dst_img_name, images[i, ...])
def _load_model(self):
config_path = f"{self.model_path}/luminoth.predict.yml"
if not os.path.exists(config_path):
self._generate_predict_luminoth_config()
config = get_config(config_path)
return PredictorNetwork(config)
def get_model_convert_output(self, model):
config = get_checkpoint_config(model)
network = PredictorNetwork(config)
output_names = []
for k, v in network.fetches.iteritems():
if k in OUTPUT_NAMES:
if isinstance(v, tf.Tensor):
output_names.append(v.name.split(':')[0])
elif isinstance(v, tuple):
for x in v:
output_names.append(x.name.split(':')[0])
logger.debug('Input name: {}\n'.format(network.image_placeholder.name))
for n in output_names:
logger.debug('Output name {}'.format(n))
return network.session, network.session.graph_def, output_names
def predict(path_or_dir, config_files, checkpoint, override_params,
output_path, save_media_to, min_prob, max_detections, only_class,
ignore_class, debug):
"""Obtain a model's predictions.
Receives either `config_files` or `checkpoint` in order to load the correct
model. Afterwards, runs the model through the inputs specified by
`path-or-dir`, returning predictions according to the format specified by
`output`.
Additional model behavior may be modified with `min-prob`, `only-class` and
`ignore-class`.
"""
if debug:
tf.logging.set_verbosity(tf.logging.DEBUG)
else:
tf.logging.set_verbosity(tf.logging.ERROR)
if only_class and ignore_class:
click.echo(
"Only one of `only-class` or `ignore-class` may be specified.")
return
# Process the input and get the actual files to predict.
files = resolve_files(path_or_dir)
if not files:
error = 'No files to predict found. Accepted formats are: {}.'.format(
', '.join(IMAGE_FORMATS + VIDEO_FORMATS))
click.echo(error)
return
else:
click.echo('Found {} files to predict.'.format(len(files)))
# Build the `Formatter` based on the outputs, which automatically writes
# the formatted output to all the requested output files.
if output_path == '-':
output = sys.stdout
else:
output = open(output_path, 'w')
# Create `save_media_to` if specified and it doesn't exist.
if save_media_to:
tf.gfile.MakeDirs(save_media_to)
# Resolve the config to use and initialize the model.
if checkpoint:
config = get_checkpoint_config(checkpoint)
elif config_files:
config = get_config(config_files)
else:
click.echo(
'Neither checkpoint not config specified, assuming `accurate`.')
config = get_checkpoint_config('accurate')
if override_params:
config = override_config_params(config, override_params)
# Filter bounding boxes according to `min_prob` and `max_detections`.
if config.model.type == 'fasterrcnn':
if config.model.network.with_rcnn:
config.model.rcnn.proposals.total_max_detections = max_detections
else:
config.model.rpn.proposals.post_nms_top_n = max_detections
config.model.rcnn.proposals.min_prob_threshold = min_prob
elif config.model.type == 'ssd':
config.model.proposals.total_max_detections = max_detections
config.model.proposals.min_prob_threshold = min_prob
else:
raise ValueError("Model type '{}' not supported".format(
config.model.type))
# Instantiate the model indicated by the config.
network = PredictorNetwork(config)
# Iterate over files and run the model on each.
for file in files:
# Get the media output path, if media storage is requested.
save_path = os.path.join(save_media_to, 'pred_{}'.format(
os.path.basename(file))) if save_media_to else None
file_type = get_file_type(file)
predictor = predict_image if file_type == 'image' else predict_video
objects = predictor(
network,
file,
only_classes=only_class,
ignore_classes=ignore_class,
save_path=save_path,
)
# TODO: Not writing jsons for video files for now.
if objects is not None and file_type == 'image':
output.write(
json.dumps({
'file': file,
'objects': objects,
}) + '\n')
output.close()
def detect_tile_cell(slide_path, tile_position, csv_dict, args, it_kwargs):
start_t = time.time()
print('--- Loading Image...')
# get slide tile source
ts = large_image.getTileSource(slide_path)
# get requested tile
tile_info = ts.getSingleTile(
tile_position=tile_position,
format=large_image.tilesource.TILE_FORMAT_NUMPY,
**it_kwargs)
# get tile image
im_tile = tile_info['tile'][:, :, :3]
t1 = time.time() - start_t
csv_dict['Image Loading'].append(round(t1, 3))
print('--- Finished Loading Image')
cv2.imwrite('hey_im_tile.png', im_tile)
# *******************************************************
#
# Perform cell detections
#
# ########################### DETECTION #################
print('--- Performing cell detections...')
config = get_config(CONFIG)
if not args.max_det is None:
config.model.rcnn.proposals.total_max_detections = args.max_det
else:
config.model.rcnn.proposals.total_max_detections = MAX_DET
if not args.min_prob is None:
config.model.rcnn.proposals.min_prob_threshold = args.min_prob
else:
config.model.rcnn.proposals.min_prob_threshold = MIN_PROB
print('--- Currently Analysing Input Image Size : ', im_tile.shape)
network = PredictorNetwork(config)
objects = network.predict_image(im_tile)
print('--- Finished Cell Detections')
t2 = time.time() - start_t
t22 = float(t2) - float(t1)
csv_dict['Cell Detection'].append(round(t22, 3))
print('***** Number of Detected Cells ****** : ', len(objects))
#
# Perform JSON loading
#
print('--- Performing Cell Crops loading...')
im_tile_rgb = cv2.cvtColor(im_tile, cv2.COLOR_BGR2RGB)
if not args.inputImageFile is None:
_, images, ids = classify_cellscropping.\
load_json(im_tile_rgb, preds=objects)
else:
_, images, ids = classify_cellscropping.\
load_json(im_tile_rgb, preds=objects)
print('--- Finished Cell Crops loading')
t3 = time.time() - start_t
t33 = float(t3) - float(t2)
csv_dict['Cell Cropping'].append(round(t33, 3))
csv_dict['Number of Cells'].append(len(ids))
# ########################### CLASSIFICATION #######################
print('--- Performing Cell Classification...')
try:
final_json = classify_main.predict(images, ids, CKPT)
except ValueError:
final_json = []
print(
'!!!!! Can not Conduct Classification on 0 Number of Cells Detected !!!!!'
)
print('--- Finished Cell Classification')
t4 = time.time() - start_t
t44 = float(t4) - float(t3)
csv_dict['Cell Classification'].append(round(t44, 3))
# # Delete border nuclei
# if args.ignore_border_nuclei is True:
# im_nuclei_seg_mask = htk_seg_label.delete_border(im_nuclei_seg_mask)
# generate cell annotations
cell_annot_list = cli_utils.create_tile_cell_annotations(
final_json, tile_info, args.cell_annotation_format)
t5 = time.time() - start_t
t55 = float(t5) - float(t4)
csv_dict['Annotation Writing'].append(round(t55, 3))
return cell_annot_list, csv_dict
from luminoth.tools.checkpoint import get_checkpoint_config
from luminoth.utils.predicting import PredictorNetwork
from PIL import Image as pilimage
# This program will predict the location of the tables in an image
# It outputs the coordinates of the tables. Using these coordinates we can cut the table portion of the image and use it for further processing
input_file = '/usr/local/table-detection-from-images-using-deep-learning-master/test_image_with_table.png'
# Specify the luminoth checkpoint here
checkpoint = 'c2df81db49e0'
config = get_checkpoint_config(checkpoint)
network = PredictorNetwork(config)
image = pilimage.open(input_file).convert('RGB')
objects = network.predict_image(image)
print("NO OF TABLES IDENTIFIED BY LUMINOTH = " + str(len(objects)))
print('-' * 100)
table_counter = 1
for i in range(len(objects)):
table_idctionary = objects[i]
coordinate_list = table_idctionary["bbox"]
xminn = coordinate_list[0]
yminn = coordinate_list[1]
xmaxx = coordinate_list[2]
ymaxx = coordinate_list[3]
print('TABLE ' + str(table_counter) + ':')
print('-' * 100)
print("xminn = " + str(xminn))
def start_network(config):
global PREDICTOR_NETWORK
PREDICTOR_NETWORK = PredictorNetwork(config)
class Detector(object):
"""Encapsulates an object detection model behavior.
In order to perform object detection with a model implemented within
Luminoth, this class should be used.
Attributes:
classes (list of str): Ordered class names for the detector.
prob (float): Default probability threshold for predictions.
TODO:
- Don't create a TF session internally (or make its creation optional)
in order to be compatible with both TF-Eager and Jupyter Notebooks.
- Manage multiple instantiations correctly in order to avoid creating
the same TF objects over and over (which appends the `_N` suffix to
the graph and makes the checkpoint loading fail).
"""
DEFAULT_CHECKPOINT = "accurate"
def __init__(self, checkpoint=None, config=None, prob=0.7, classes=None):
"""Instantiate a detector object with the appropriate config.
Arguments:
checkpoint (str): Checkpoint id or alias to instantiate the
detector as.
config (dict): Configuration parameters describing the desired
model. See `get_config` to load a config file.
Note:
Only one of the parameters must be specified. If none is, we
default to loading the checkpoint indicated by
`DEFAULT_CHECKPOINT`.
"""
if checkpoint is not None and config is not None:
raise ValueError(
"Only one of `checkpoint` or `config` must be specified in "
"order to instantiate a Detector.")
if checkpoint is None and config is None:
# Neither checkpoint no config specified, default to
# `DEFAULT_CHECKPOINT`.
checkpoint = self.DEFAULT_CHECKPOINT
if checkpoint:
config = get_checkpoint_config(checkpoint)
# Prevent the model itself from filtering its proposals (default
# value of 0.5 is in use in the configs).
# TODO: A model should always return all of its predictions. The
# filtering should be done (if at all) by PredictorNetwork.
if config.model.type == "fasterrcnn":
config.model.rcnn.proposals.min_prob_threshold = 0.0
elif config.model.type == "ssd":
config.model.proposals.min_prob_threshold = 0.0
# TODO: Remove dependency on `PredictorNetwork` or clearly separate
# responsibilities.
self._network = PredictorNetwork(config)
self.prob = prob
# Use the labels when available, integers when not.
self._model_classes = (self._network.class_labels
if self._network.class_labels else list(
range(config.model.network.num_classes)))
if classes:
self.classes = set(classes)
if not set(self._model_classes).issuperset(self.classes):
raise ValueError(
"`classes` must be contained in the detector's classes. "
"Available classes are: {}.".format(self._model_classes))
else:
self.classes = set(self._model_classes)
def predict(self, images, prob=None, classes=None):
"""Run the detector through a set of images.
Arguments:
images (numpy.ndarray or list): Either array of dimensions
`(height, width, channels)` (single image) or array of
dimensions `(number_of_images, height, width, channels)`
(multiple images). If a list, must be a list of rank 3 arrays.
prob (float): Override configured probability threshold for
predictions.
classes (set of str): Override configured class names to consider.
Returns:
Either list of objects detected in the image (single image case) or
list of list of objects detected (multiple images case).
In the multiple images case, the outer list has `number_of_images`
elements, while the inner ones have the number of objects detected
in each image.
Each object has the format::
{
'bbox': [x_min, y_min, x_max, y_max],
'label': '<cat|dog|person|...>' | 0..C,
'prob': prob
}
The coordinates are integers, where `(x_min, y_min)` are the
coordinates of the top-left corner of the bounding box, while
`(x_max, y_max)` the bottom-right. By convention, the top-left
corner of the image is coordinate `(0, 0)`.
The probability, `prob`, is a float between 0 and 1, indicating the
confidence of the detection being correct.
The label of the object, `label`, may be either a string if the
classes file for the model is found or an integer between 0 and the
number of classes `C`.
"""
# If it's a single image (ndarray of rank 3), turn into a list.
single_image = False
if not isinstance(images, list):
if len(images.shape) == 3:
images = [images]
single_image = True
if prob is None:
prob = self.prob
if classes is None:
classes = self.classes
else:
classes = set(classes)
# TODO: Remove the loop once (and if) we implement batch sizes. Neither
# Faster R-CNN nor SSD support batch size yet, so it's the same for
# now.
predictions = []
for image in images:
predictions.append([
pred for pred in self._network.predict_image(image)
if pred["prob"] >= prob and pred["label"] in classes
])
if single_image:
predictions = predictions[0]
return predictions
def create_inference_server(self, config):
return PredictorNetwork(config)
def load_detector():
config = get_checkpoint_config(settings.LUMI_CHECKPOINT)
detector = PredictorNetwork(config)
return detector
def predict(path_or_dir, config_files, checkpoint, override_params, output_dir,
save, min_prob, ignore_classes, debug):
if debug:
tf.logging.set_verbosity(tf.logging.DEBUG)
else:
tf.logging.set_verbosity(tf.logging.INFO)
# Get file paths
if tf.gfile.IsDirectory(path_or_dir):
file_paths = [
os.path.join(path_or_dir, f)
for f in tf.gfile.ListDirectory(path_or_dir)
if get_filetype(f) in ('image', 'video')
]
else:
if get_filetype(path_or_dir) in ('image', 'video'):
file_paths = [path_or_dir]
else:
file_paths = []
errors = 0
successes = 0
created_files_paths = []
total_files = len(file_paths)
if total_files == 0:
no_files_message = ("No images or videos found. "
"Accepted formats -> Image: {} - Video: {}")
tf.logging.error(no_files_message.format(IMAGE_FORMATS, VIDEO_FORMATS))
exit()
# Resolve the config to use and initialize the mdoel.
if checkpoint:
config = get_checkpoint_config(checkpoint)
elif config_files:
config = get_config(config_files)
else:
click.echo('You must specify either a checkpoint or a config file.')
exit()
if override_params:
config = override_config_params(config, override_params)
network = PredictorNetwork(config)
# Create output_dir if it doesn't exist
if output_dir:
tf.gfile.MakeDirs(output_dir)
tf.logging.info('Getting predictions for {} files'.format(total_files))
# Iterate over file paths
for file_path in file_paths:
save_path = 'pred_' + os.path.basename(file_path)
if output_dir:
save_path = os.path.join(output_dir, save_path)
if get_filetype(file_path) == 'image':
click.echo('Predicting {}...'.format(file_path))
with tf.gfile.Open(file_path, 'rb') as f:
try:
image = Image.open(f).convert('RGB')
except (tf.errors.OutOfRangeError, OSError) as e:
tf.logging.warning('Error: {}'.format(e))
tf.logging.warning("Couldn't open: {}".format(file_path))
errors += 1
continue
# Run image through network
prediction = network.predict_image(image)
successes += 1
# Filter results if required by user
if ignore_classes:
prediction = filter_classes(prediction, ignore_classes)
# Save prediction json file
with open(save_path + '.json', 'w') as outfile:
json.dump(prediction, outfile)
created_files_paths.append(save_path + '.json')
# Save predicted image
if save:
with tf.gfile.Open(file_path, 'rb') as im_file:
image = Image.open(im_file)
draw_bboxes_on_image(image, prediction, min_prob)
image.save(save_path)
created_files_paths.append(save_path)
elif get_filetype(file_path) == 'video':
# NOTE: We'll hardcode the video ouput to mp4 for the time being
save_path = os.path.splitext(save_path)[0] + '.mp4'
try:
writer = skvideo.io.FFmpegWriter(save_path)
except AssertionError as e:
tf.logging.error(e)
tf.logging.error(
"Please install ffmpeg before making video predictions.")
exit()
num_of_frames = int(
skvideo.io.ffprobe(file_path)['video']['@nb_frames'])
video_progress_bar = click.progressbar(
skvideo.io.vreader(file_path),
length=num_of_frames,
label='Predicting {}'.format(file_path))
with video_progress_bar as bar:
try:
for frame in bar:
# Run image through network
prediction = network.predict_image(frame)
# Filter results if required by user
if ignore_classes:
prediction = filter_classes(
prediction, ignore_classes)
image = Image.fromarray(frame)
draw_bboxes_on_image(image, prediction, min_prob)
writer.writeFrame(np.array(image))
except RuntimeError as e:
click.echo() # Error prints next to progress-bar if not
tf.logging.error('Error: {}'.format(e))
tf.logging.error('Corrupt videofile: {}'.format(file_path))
tf.logging.error(
'Partially processed video file saved in {}'.format(
save_path))
errors += 1
writer.close()
created_files_paths.append(save_path)
else:
tf.logging.warning("{} isn't an image/video".format(file_path))
# Generate logs
tf.logging.info("Created the following files: {}".format(
', '.join(created_files_paths)))
if errors:
tf.logging.warning('{} errors.'.format(errors))
