def _download_project_optimized(api: Api,
project_id,
project_dir,
datasets_whitelist=None,
cache=None,
progress_cb=None):
project_info = api.project.get_info_by_id(project_id)
project_id = project_info.id
logger.info(
f"Annotations are not cached (always download latest version from server)"
)
project_fs = Project(project_dir, OpenMode.CREATE)
meta = ProjectMeta.from_json(api.project.get_meta(project_id))
project_fs.set_meta(meta)
for dataset_info in api.dataset.get_list(project_id):
dataset_name = dataset_info.name
dataset_id = dataset_info.id
need_download = True
if datasets_whitelist is not None and dataset_id not in datasets_whitelist:
need_download = False
if need_download is True:
dataset = project_fs.create_dataset(dataset_name)
_download_dataset(api,
dataset,
dataset_id,
cache=cache,
progress_cb=progress_cb)
def _report_metrics(m_type, epoch, metrics):
logger.info('metrics', extra={
'event_type': EventType.METRICS,
'type': m_type,
'epoch': epoch,
'metrics': metrics
})
def print_evaluation_values(tag_pair_metrics):
labels = [
ACCURACY, PRECISION, RECALL, F1_MEASURE, TRUE_POSITIVE,
TRUE_NEGATIVE, FALSE_POSITIVE, FALSE_NEGATIVE
]
for label in labels:
logger.info(' {0}: {1:2.4f}'.format(
label.ljust(16), tag_pair_metrics[label]))
def report_checkpoint_saved(checkpoint_idx, subdir, sizeb, best_now, optional_data):
logger.info('checkpoint', extra={
'event_type': EventType.CHECKPOINT,
'id': checkpoint_idx,
'subdir': subdir,
'sizeb': sizeb,
'best_now': best_now,
'optional': optional_data
})
def _construct_and_fill_model(self):
super()._construct_and_fill_model()
self.device_ids = sly.env.remap_gpu_devices([self._config[GPU_DEVICE]])
n_cls = (max(self.out_class_mapping.keys()) + 1)
self.model = create_model_for_inference(n_cls=n_cls,
device_ids=self.device_ids,
model_dir=TaskPaths.MODEL_DIR)
logger.info('Weights are loaded.')
def _construct_and_fill_model(self):
super()._construct_and_fill_model()
device_ids = sly.env.remap_gpu_devices([self._config[GPU_DEVICE]])
num_layers = determine_resnet_model_configuration(TaskPaths.MODEL_CONFIG_PATH)
self.model = create_model(num_layers=num_layers, n_cls=(max(self.classification_tags_to_idx.values()) + 1),
device_ids=device_ids)
self.model = WeightsRW(TaskPaths.MODEL_DIR).load_strictly(self.model)
self.model.eval()
logger.info('Weights are loaded.')
def report_dtl_verification_finished(output):
'''
Logs a message with level INFO on logger
:param output: str
'''
logger.info('Verification finished.',
extra={
'output': output,
'event_type': EventType.TASK_VERIFIED
})
def get_metrics(self): # Macro-evaluation
logger.info('Start evaluation of macro metrics.')
result = {
gt_class: {
AP:
self._calculate_average_precision(
gt_class, self._gt_to_pred_class_mapping[gt_class],
pair_counters)
}
for gt_class, pair_counters in self._counters.items()
}
logger.info('Finish macro evaluation')
return result
def _load_train_config(self): # @TODO: partly copypasted from SingleImageInferenceBase
self._load_raw_model_config_json()
self.classification_tags = self._model_out_img_tags()
logger.info('Read model out tags', extra={'tags': self.classification_tags.to_json()})
self.classification_tags_to_idx = self.train_config[self.classification_tags_to_idx_key]
logger.info('Read model internal tags mapping', extra={'tags_mapping': self.classification_tags_to_idx})
self._model_out_meta = ProjectMeta(obj_classes=ObjClassCollection(),
img_tag_metas=self.classification_tags,
obj_tag_metas=self._model_out_obj_tags())
self.idx_to_classification_tags = {v: k for k, v in self.classification_tags_to_idx.items()}
self._determine_model_input_size()
def _check_projects_compatible_structure(self):
if self._project_gt.datasets.keys() != self._project_pred.datasets.keys(): # Keys is sorted - ok
raise RuntimeError('Projects must contain same datasets.')
if self._project_gt.total_items != self._project_pred.total_items:
raise RuntimeError('Projects must contain same number of samples.')
for ds_gt in self._project_gt.datasets:
ds_pred = self._project_pred.datasets.get(ds_gt.name)
for sample_name in ds_gt:
if not ds_pred.item_exists(sample_name):
raise RuntimeError('Projects must contain identically named samples in respective datasets. ' +
'Ground truth project has sample {!r} in dataset {!r}, but prediction project ' +
'does not.'.format(sample_name, ds_gt.name))
logger.info('Projects structure has been read. Samples: {} per project.'.format(self._project_gt.total_items))
def _construct_and_fill_model(self):
super()._construct_and_fill_model()
self.device_ids = sly.env.remap_gpu_devices([self._config[GPU_DEVICE]])
n_cls = (max(self.out_class_mapping.keys()) + 1)
use_batchnorm = self.train_config[SETTINGS]['use_batchnorm']
model = ICNet(n_classes=n_cls,
input_size=self.input_size,
is_batchnorm=use_batchnorm)
WeightsRW(TaskPaths.MODEL_DIR).load_strictly(model)
logger.info('Weights are loaded.')
self.model = DataParallel(model, device_ids=self.device_ids)
self.model.eval()
self.model.cuda()
def log_total_metrics(self):
log_line()
log_head(' Result metrics values for {} IoU threshold '.format(self._iou_threshold))
classes_values = self.get_metrics()
for i, (cls_gt, pair_values) in enumerate(classes_values.items()):
average_precision = pair_values[AP]
log_line()
log_head(' Results for pair of classes <<{} <-> {}>> '.format(cls_gt,
self._gt_to_pred_class_mapping[cls_gt]))
logger.info('Average Precision (AP): {}'.format(average_precision))
log_line()
log_head(' Mean metrics values ')
logger.info('Mean Average Precision (mAP): {}'.format(self.average_per_class_avg_precision(classes_values)))
log_line()
def log_total_metrics(self):
logger.info('**************** Result IoU metric values ****************')
logger.info('NOTE! Values for "intersection" and "union" are in pixels.')
for i, (cls_gt, values) in enumerate(self.get_metrics().items(), start=1):
iou_line = _iou_log_line(values[IOU], values[INTERSECTION], values[UNION])
logger.info('{}. Classes {} <-> {}: {}'.format(i, cls_gt, self._class_mapping[cls_gt], iou_line))
total_values = self.get_total_metrics()
logger.info(
'Total: {}'.format(_iou_log_line(total_values[IOU], total_values[INTERSECTION], total_values[UNION])))
def log_total_metrics(self):
def exp_one(arg):
return str(arg).center(20)
def exp_arg(args):
return [exp_one(arg) for arg in args]
log_line()
log_head(' Result metrics values for {} IoU threshold '.format(
self._iou_threshold))
log_head(' Confusion matrix ')
sorted_gt_names = sorted(self._class_mapping.keys())
pred_names = [
self._class_mapping[gt_name] for gt_name in sorted_gt_names
]
logger.info(''.join(exp_arg([''] + pred_names + ['False Negatives'])))
for gt_name in sorted_gt_names:
logger.info(''.join([exp_one(gt_name)] + exp_arg([
self._confusion_matrix[gt_name, pred_name]
for pred_name in pred_names
]) + [exp_one(self._unmatched_gt[gt_name])]))
log_line()
logger.info(''.join([exp_one('False Positives')] + exp_arg(
[self._unmatched_pred[pred_name]
for pred_name in pred_names]) + [exp_one('0')]))
log_line()
def log_total_metrics(self):
log_line()
log_head(' Result metrics values for {} IoU threshold '.format(self._iou_threshold))
for i, (gt_class, values) in enumerate(self.get_metrics().items()):
log_line()
log_head(' Results for pair of classes <<{} <-> {}>> '.format(gt_class,
self._gt_to_pred_class_mapping[gt_class]))
logger.info('Precision: {}'.format(values[PRECISION]))
logger.info('Recall: {}'.format(values[RECALL]))
log_line()
log_head(' Total metrics values ')
total_values = self.get_total_metrics()
logger.info('Precision: {}'.format(total_values[PRECISION]))
logger.info('Recall: {}'.format(total_values[RECALL]))
log_line()
def create_model(num_layers, n_cls, device_ids):
logger.info('Will construct ResNet{} model.'.format(num_layers))
model = num_layers_to_model[num_layers]()
model.avgpool = nn.AdaptiveAvgPool2d((1, 1))
in_features = model.fc.in_features
model.fc = nn.Linear(in_features, n_cls)
logger.info('Model has been constructed (w/out weights).')
model = DataParallel(model, device_ids=device_ids).cuda()
logger.info('Model has been loaded into GPU(s).', extra={'remapped_device_ids': device_ids})
return model
def _construct_and_fill_model(self):
super()._construct_and_fill_model()
self.device_ids = sly.env.remap_gpu_devices([self._config[GPU_DEVICE]])
self.graph, self.model = inference_lib.construct_and_fill_model(
self.class_title_to_idx, self.input_size_limits,
TaskPaths.MODEL_DIR)
logger.info('Weights are loaded.')
logger.info('Warming up the model with a dummy image.')
with self.graph.as_default():
self.model.detect([np.zeros([256, 256, 3], dtype=np.uint8)],
verbose=0)
logger.info('Model warmup finished.')
def log_total_metrics(self):
common_info = """
P = condition positive (the number of real positive cases in the data)
N = condition negative (the number of real negative cases in the data)
TP = True Positive prediction
TN = True Negative prediction
FP = False Positive prediction (Type I error)
FN = False Negative prediction (Type II error)
Accuracy = (TP + TN)/(TP + TN + FP + FN) = TRUE/TOTAL
Precision = TP / (TP + FP)
Recall = TP / (TP + FN)
F1-Measure = (2 * TP) / (2 * TP + FP + FN)
"""
log_line()
log_line(c='*')
for line in common_info.split('\n'):
line = line.strip()
if len(line) > 0:
logger.info(line.ljust(80))
log_line(c='*')
log_line()
def print_evaluation_values(tag_pair_metrics):
labels = [
ACCURACY, PRECISION, RECALL, F1_MEASURE, TRUE_POSITIVE,
TRUE_NEGATIVE, FALSE_POSITIVE, FALSE_NEGATIVE
]
for label in labels:
logger.info(' {0}: {1:2.4f}'.format(
label.ljust(16), tag_pair_metrics[label]))
for i, (tag_name_gt,
tag_metrics) in enumerate(self.get_metrics().items(), start=1):
logger.info('{}) {} <--> {}:'.format(
i, tag_name_gt, self._tags_mapping[tag_name_gt]))
print_evaluation_values(tag_metrics)
log_line()
logger.info('Total values:')
total_values = self.get_total_metrics()
print_evaluation_values(total_values)
log_line()
log_line(c='*')
def _download_dataset(api: Api,
dataset,
dataset_id,
cache=None,
progress_cb=None):
images = api.image.get_list(dataset_id)
images_to_download = images
# copy images from cache to task folder and download corresponding annotations
if cache:
images_to_download, images_in_cache, images_cache_paths = _split_images_by_cache(
images, cache)
if len(images_to_download) + len(images_in_cache) != len(images):
raise RuntimeError(
"Error with images cache during download. Please contact support."
)
logger.info(f"Download dataset: {dataset.name}",
extra={
"total": len(images),
"in cache": len(images_in_cache),
"to download": len(images_to_download)
})
if len(images_in_cache) > 0:
img_cache_ids = [img_info.id for img_info in images_in_cache]
ann_info_list = api.annotation.download_batch(
dataset_id, img_cache_ids, progress_cb)
img_name_to_ann = {
ann.image_id: ann.annotation
for ann in ann_info_list
}
for batch in batched(list(zip(images_in_cache,
images_cache_paths)),
batch_size=50):
for img_info, img_cache_path in batch:
item_name = _maybe_append_image_extension(
img_info.name, img_info.ext)
dataset.add_item_file(item_name,
img_cache_path,
img_name_to_ann[img_info.id],
_validate_item=False,
_use_hardlink=True)
progress_cb(len(batch))
# download images from server
if len(images_to_download) > 0:
# prepare lists for api methods
img_ids = []
img_paths = []
for img_info in images_to_download:
img_ids.append(img_info.id)
# TODO download to a temp file and use dataset api to add the image to the dataset.
img_paths.append(
os.path.join(
dataset.img_dir,
_maybe_append_image_extension(img_info.name,
img_info.ext)))
# download annotations
ann_info_list = api.annotation.download_batch(dataset_id, img_ids,
progress_cb)
img_name_to_ann = {
ann.image_id: ann.annotation
for ann in ann_info_list
}
api.image.download_paths(dataset_id, img_ids, img_paths, progress_cb)
for img_info, img_path in zip(images_to_download, img_paths):
dataset.add_item_file(img_info.name, img_path,
img_name_to_ann[img_info.id])
if cache:
img_hashes = [img_info.hash for img_info in images_to_download]
cache.write_objects(img_paths, img_hashes)
def get_metrics(self): # Macro-evaluation
logger.info('Start evaluation of macro metrics.')
result = {pair_name: {AP: self._calculate_average_precision(pair_name, pair_counters)}
for pair_name, pair_counters in self._counters.items()}
logger.info('Finish macro evaluation')
return result
def report_import_finished():
logger.info('import finished',
extra={'event_type': EventType.IMPORT_APPLIED})
def log_head(string):
logger.info(string.center(80, '*'))
def log_line(length=80, c=' '):
logger.info(c * length)
def report_inference_finished():
logger.info('model applied', extra={'event_type': EventType.MODEL_APPLIED})
def report_dtl_finished():
'''
Logs a message with level INFO on logger
'''
logger.info('DTL finished', extra={'event_type': EventType.DTL_APPLIED})
def report_inference_finished():
'''
Logs a message with level INFO on logger
'''
logger.info('model applied', extra={'event_type': EventType.MODEL_APPLIED})
def report_import_finished():
'''
Logs a message with level INFO on logger
'''
logger.info('import finished',
extra={'event_type': EventType.IMPORT_APPLIED})
def report_agent_rpc_ready():
'''
Logs a message with level INFO on logger
'''
logger.info('Ready to get events',
extra={'event_type': EventType.TASK_DEPLOYED})
def report_dtl_finished():
logger.info('DTL finished', extra={'event_type': EventType.DTL_APPLIED})
def report_dtl_verification_finished(output):
logger.info('Verification finished.',
extra={
'output': output,
'event_type': EventType.TASK_VERIFIED
})