class SendReviewReports(object):
def __init__(self, host, user, passwd, logger):
self.promort_client = ProMortClient(host, user, passwd)
self.logger = logger
def _send_reports(self, receiver):
url = 'api/odin/reviews_activity_report/send/'
response = self.promort_client.get(url, payload={'receiver': receiver})
return response.json()
def run(self, receiver):
self.promort_client.login()
try:
self._send_reports(receiver)
self.logger.info('Report sent to %s', receiver)
except (UserNotAllowed, ProMortInternalServerError) as e:
self.logger.error(e.message)
self.promort_client.logout()
except ProMortAuthenticationError, e:
self.logger.error(e)
class CasesOverallScoring(object):
def __init__(self, host, user, passwd, logger):
self.promort_client = ProMortClient(host, user, passwd)
self.logger = logger
def _get_cases(self):
url = 'api/cases/'
response = self.promort_client.get(url)
if response.status_code == requests.codes.OK:
return [(c['id'], c['laboratory']) for c in response.json()]
return []
def _get_case_overall_score(self, case_id):
url = 'api/odin/reviews/%s/score/' % case_id
response = self.promort_client.get(url)
if response.status_code == requests.codes.OK:
return response.json().values()
return None
def run(self, out_file):
self.promort_client.login()
try:
cases = self._get_cases()
with open(out_file, 'w') as output_file:
writer = csv.DictWriter(
output_file,
['case', 'laboratory', 'primary_score', 'secondary_score'])
writer.writeheader()
for case, lab in cases:
scores = self._get_case_overall_score(case)
if scores is not None:
for score in scores:
score['case'] = case
score['laboratory'] = lab
writer.writerow(score)
self.promort_client.logout()
except UserNotAllowed, e:
self.logger.error(e.message)
self.promort_client.logout()
except ProMortAuthenticationError, e:
self.logger.error(e.message)
def __init__(self, host, user, passwd, logger):
self.promort_client = ProMortClient(host, user, passwd)
self.logger = logger
def __init__(self, host, user, passwd, logger):
self.promort_client = ProMortClient(host, user, passwd)
self.shapes_manager = ShapesManager(self.promort_client)
self.logger = logger
class RandomPatchesExtractor(object):
def __init__(self, host, user, passwd, logger):
self.promort_client = ProMortClient(host, user, passwd)
self.shapes_manager = ShapesManager(self.promort_client)
self.logger = logger
def _build_data_mappings(self, focus_regions_list):
dependencies_tree = dict()
positive_focus_regions = set()
negative_focus_regions = set()
with open(focus_regions_list) as f:
reader = DictReader(f)
for row in reader:
dependencies_tree.setdefault(row['slide_id'], dict())\
.setdefault(row['parent_core_id'], list()).append(row['focus_region_id'])
if row['tissue_status'] == 'TUMOR':
positive_focus_regions.add(row['focus_region_id'])
elif row['tissue_status'] == 'NORMAL':
negative_focus_regions.add(row['focus_region_id'])
return dependencies_tree, positive_focus_regions, negative_focus_regions
def _load_focus_regions(self, focus_regions, slide_id, positive_regions,
negative_regions):
fregions = {'positive': [], 'negative': []}
for region in focus_regions:
if region in positive_regions:
fregions['positive'].append(
(self.shapes_manager.get_focus_region(slide_id,
region), region))
elif region in negative_regions:
fregions['negative'].append(
(self.shapes_manager.get_focus_region(slide_id,
region), region))
else:
self.logger.critical(
'There is no classification for focus region %r of slide %s',
region, slide_id)
return fregions
def _extract_patch(self, point, scaling, extractor):
return extractor.get_patch((point.x, point.y), scaling)
def _get_tissue_masks(self, patch_coordinates, core, scaling, tolerance):
tissue_mask = core.get_intersection_mask(patch_coordinates, scaling,
tolerance)
not_tissue_mask = core.get_difference_mask(patch_coordinates, scaling,
tolerance)
return tissue_mask, not_tissue_mask
def _get_regions_mask(self, patch_coordinates, regions, tile_size, scaling,
tolerance):
mask = np.zeros((tile_size, tile_size), np.uint8)
for r in regions:
m = r[0].get_intersection_mask(patch_coordinates, scaling,
tolerance)
mask = mmu.add_mask(mask, m)
return mask
def _get_positive_regions_mask(self, patch_coordinates, positive_regions,
tile_size, scaling, tolerance):
return self._get_regions_mask(patch_coordinates, positive_regions,
tile_size, scaling, tolerance)
def _get_negative_regions_mask(self, patch_coordinates, negative_regions,
tile_size, scaling, tolerance):
return self._get_regions_mask(patch_coordinates, negative_regions,
tile_size, scaling, tolerance)
def _build_masks(self, patch_coordinates, core, positive_regions,
negative_regions, patch_image, tile_size, scaling,
tolerance, white_lower_bound):
tissue_mask, not_tissue_mask = self._get_tissue_masks(
patch_coordinates, core, scaling, tolerance)
return {
'tissue':
tissue_mask,
'not_tissue':
not_tissue_mask,
'tumor':
self._get_positive_regions_mask(patch_coordinates,
positive_regions, tile_size,
scaling, tolerance),
'not_tumor':
self._get_negative_regions_mask(patch_coordinates,
negative_regions, tile_size,
scaling, tolerance),
'cv2_white':
extract_white_mask(patch_image, white_lower_bound)
}
def _serialize_patch(self, patch_img, slide_id, output_folder):
f_uuid = uuid4().hex
try:
os.makedirs(os.path.join(output_folder, slide_id))
except OSError:
pass
out_file = os.path.join(output_folder, slide_id, '%s.jpeg' % f_uuid)
patch_img.save(out_file)
return f_uuid
def _serialize_masks(self, masks, patch_uuid, slide_id, output_folder):
out_file = os.path.join(output_folder, slide_id, '%s.npz' % patch_uuid)
np.savez_compressed(out_file,
tissue=masks['tissue'],
not_tissue=masks['not_tissue'],
tumor=masks['tumor'],
not_tumor=masks['not_tumor'],
cv2_white=masks['cv2_white'])
def _serialize(self, patch, masks, slide_id, output_folder):
patch_uuid = self._serialize_patch(patch, slide_id, output_folder)
self._serialize_masks(masks, patch_uuid, slide_id, output_folder)
return patch_uuid
def _save_slide_map(self, slide_id, slide_map, output_folder):
out_file = os.path.join(output_folder, slide_id, 'patches_map.csv')
with open(out_file, 'w') as ofile:
writer = DictWriter(ofile,
['slide_id', 'focus_region_id', 'patch_uuid'])
writer.writeheader()
for row in slide_map:
writer.writerow(row)
def _patches_folder_exists(self, slide_id, output_folder):
return os.path.isdir(os.path.join(output_folder, slide_id))
def run(self, focus_regions_list, slides_folder, tile_size, patches_count,
scaling, tolerance, white_lower_bound, output_folder):
try:
self.promort_client.login()
dependencies_tree, positive_regions, negative_regions = self._build_data_mappings(
focus_regions_list)
for slide, cores in dependencies_tree.iteritems():
if not self._patches_folder_exists(slide, output_folder):
slide_path = os.path.join(slides_folder, '%s.mrxs' % slide)
slide_map = list()
self.logger.info('Processing file %s', slide_path)
patches_extractor = PatchesExtractor(
DeepZoomWrapper(slide_path, tile_size))
for core, focus_regions in cores.iteritems():
core_shape = self.shapes_manager.get_core(slide, core)
self.logger.info('Loading core %s', core)
focus_regions_shapes = self._load_focus_regions(
focus_regions, slide, positive_regions,
negative_regions)
self.logger.info(
'Loaded %d positive shapes and %d negative',
len(focus_regions_shapes['positive']),
len(focus_regions_shapes['negative']))
for focus_region in chain(
*focus_regions_shapes.values()):
try:
for point in focus_region[0].get_random_points(
patches_count):
processed = False
tolerance_value = 0.0
while not processed:
try:
patch, coordinates = self._extract_patch(
point, scaling,
patches_extractor)
masks = self._build_masks(
coordinates, core_shape,
focus_regions_shapes[
'positive'],
focus_regions_shapes[
'negative'], patch,
tile_size, scaling,
tolerance_value,
white_lower_bound)
patch_uuid = self._serialize(
patch, masks, slide,
output_folder)
slide_map.append({
'slide_id':
slide,
'focus_region_id':
focus_region[1],
'patch_uuid':
patch_uuid
})
processed = True
except TopologicalError:
tolerance_value += tolerance
self.logger.debug(
'Intersection failed, increasing tolerance to %f',
tolerance_value)
except DZIBadTileAddress, e:
self.logger.error(e.message)
processed = True
except InvalidPolygonError:
self.logger.error(
'FocusRegion is not a valid shape, skipping it'
)
try:
self._save_slide_map(slide, slide_map, output_folder)
except IOError:
self.logger.warning(
'There is no output folder for slide %s, no focus regions map to save',
slide)
else:
self.logger.warning(
'There is already a patches folder for slide %s, skipping it',
slide)
self.promort_client.logout()
except UserNotAllowed, e:
self.logger.error('UserNotAllowedError: %r', e.message)
self.promort_client.logout()
except ProMortAuthenticationError, e:
self.logger.error('AuthenticationError: %r', e.message)
def __init__(self, host, user, password, log_level='INFO', log_file=None):
self.promort_client = ProMortClient(host, user, password)
self.logger = self._get_logger(log_level, log_file)
class ROIsApplier(object):
def __init__(self, host, user, password, log_level='INFO', log_file=None):
self.promort_client = ProMortClient(host, user, password)
self.logger = self._get_logger(log_level, log_file)
def _get_logger(self, log_level='INFO', log_file=None, mode='a'):
LOG_FORMAT = '%(asctime)s|%(levelname)-8s|%(message)s'
LOG_DATEFMT = '%Y-%m-%d %H:%M:%S'
logger = logging.getLogger('mask_applier')
if not isinstance(log_level, int):
try:
log_level = getattr(logging, log_level)
except AttributeError:
raise ValueError('Unsupported literal log level: %s' % log_level)
logger.setLevel(log_level)
logger.handlers = []
if log_file:
handler = logging.FileHandler(log_file, mode=mode)
else:
handler = logging.StreamHandler()
formatter = logging.Formatter(LOG_FORMAT, datefmt=LOG_DATEFMT)
handler.setFormatter(formatter)
logger.addHandler(handler)
return logger
def _get_slide_label(self, slide_path):
slide_fname = slide_path.split('/')[-1]
return slide_fname.split('.')[0]
def _get_rois_points(self, roi_json):
return [(s['point']['x'], s['point']['y']) for s in roi_json]
def _load_rois(self, query_url):
self.promort_client.login()
response = self.promort_client.get(query_url, None)
if response.status_code == 200:
rois = [(
self._get_rois_points(json.loads(fr['roi_json'])['segments']),
fr.get('tissue_status')
)
for fr in response.json()]
else:
self.logger.error('ERROR %d while retrieving ROIs', response.status_code)
rois = []
self.promort_client.logout()
return rois
def _load_slices(self, slide_id):
self.logger.info('Retrieving slices')
return self._load_rois('api/odin/rois/%s/slices/' % slide_id)
def _load_cores(self, slide_id):
self.logger.info('Retrieving cores')
return self._load_rois('api/odin/rois/%s/cores/' % slide_id)
def _load_focus_regions(self, slide_id):
self.logger.info('Retrieving focus regions')
return self._load_rois('api/odin/rois/%s/focus_regions/' % slide_id)
def _get_scaled_shape(self, roi_json, scale_factor):
self.logger.debug('Rescaling ROI')
shape = Shape(roi_json)
scaled_shape = shape._rescale_polygon(scale_factor)
return mapping(scaled_shape)['coordinates']
def _save_new_image(self, image, slide_label, output_path):
out_path = os.path.join(output_path, '%s.jpeg' % slide_label)
self.logger.debug('Saving image to %s', out_path)
image.save(out_path)
def _draw_roi(self, image, roi_json, zoom_level, line_color, line_width=5):
scaled_roi = self._get_scaled_shape(roi_json[0], zoom_level)
image.line(list(scaled_roi[0]), fill=line_color, width=line_width)
def _apply_rois(self, original_slide, slices, cores, focus_regions, zoom_level, slide_label, output_path):
image = Image.open(original_slide)
draw = ImageDraw.Draw(image)
self.logger.info('Applying slices')
for rj in slices:
self._draw_roi(draw, rj, zoom_level, 'black')
self.logger.info('Applying cores')
for rj in cores:
self._draw_roi(draw, rj, zoom_level, 'blue')
self.logger.info('Applying focus regions')
tissue_color_map = {
'TUMOR': 'red',
'NORMAL': 'green',
'STRESSED': 'orange'
}
for rj in focus_regions:
self._draw_roi(draw, rj, zoom_level, tissue_color_map[rj[1]])
self._save_new_image(image, slide_label, output_path)
def run(self, original_slide, zoom_level, output_path):
slide_label = self._get_slide_label(original_slide)
self.logger.info('Processing ROIs for slide %s', slide_label)
slices = self._load_slices(slide_label)
cores = self._load_cores(slide_label)
focus_regions = self._load_focus_regions(slide_label)
self._apply_rois(original_slide, slices, cores, focus_regions, zoom_level, slide_label, output_path)
self.logger.info('Job completed')