def Detect(self, request, context):
""" real time detection task!
"""
t1 = time.perf_counter()
anypb = Any()
height = request.height
width = request.width
image = np.frombuffer(request.image, dtype=np.uint8)
#if image.ndim == 1:
# image = image.reshape([height,width])
if not request.isRaw:
image = cv2.imdecode(image, cv2.IMREAD_GRAYSCALE)
height, width = image.shape
image = image.reshape([height, width])
# check image size, single-channel gray image
if 0 == image.size or image.size != height * width:
logging.error(
f'image shape is {image.shape}, but height = {height}, width = {width}'
)
return thyroidrpc_pb2.ProtoResponse(
code=WRONG_IMAGE_SHAPE,
msg=ErrorDict[WRONG_IMAGE_SHAPE],
data=anypb)
print(image.shape)
if image.ndim == 1:
image = image.reshape([height, width])
res = pymrcnn.ThyroidAIDoInference(image, height, width)
nodule_nums = len(res)
logging.info(f'nodules_nums: {nodule_nums}')
if 0 == nodule_nums:
nodules = thyroidrpc_pb2.Nodules(nums=0, nodule=[])
else:
nodules_list = []
for i in range(nodule_nums):
nodules_list.append(
thyroidrpc_pb2.NoduleWithNum(n=i + 1,
m=0,
x=res[i][0],
y=res[i][1],
w=res[i][2],
h=res[i][3],
s=0,
pos=0))
nodules = thyroidrpc_pb2.Nodules(nums=nodule_nums,
nodule=nodules_list)
print(nodules)
# packing results to any
anypb.Pack(nodules)
t2 = time.perf_counter()
logging.info('tensorrt: {:.2f}'.format((t2 - t1) * 1000))
return thyroidrpc_pb2.ProtoResponse(code=0, msg='', data=anypb)
def DeleteNodule(self, request, context):
""" delete nodules in one image
"""
logging.info('---------------------------------------')
logging.info(
f'deleting nodules, uid: {request.uid}, scanID: {request.scanID}, imageID: {request.imageID}'
)
logging.info(f'total requst number: {len(request.noduleID)}')
for nindex in request.noduleID:
res = userdb.collection.delete_one({
'uid': request.uid,
'scanID': request.scanID,
'imageID': request.imageID,
'nodule': nindex
})
if res.deleted_count != 1:
logging.error(f'deleting nodule fail at: {nindex}')
return thyroidrpc_pb2.ProtoResponse(code=0, msg='', data=Any())
else:
logging.info(f'deleting nodule success: {nindex}')
userdb.collection.update_many(
{
'uid': request.uid,
'scanID': request.scanID,
'nodule': nindex,
'flag_report': True
}, {'$set': {
'flag_report': False
}})
userdb.report.delete_one({
'uid': request.uid,
'scanID': request.scanID,
'nodule': nindex
})
return thyroidrpc_pb2.ProtoResponse(code=0, msg='', data=Any())
def ReadReport(self, request, context):
""" read final report
"""
logging.info('---------------------------------------')
logging.info('ReadReport')
t1 = time.perf_counter()
# output list
list_ds = []
list_nodule = []
list_benign_prob = []
list_tirads = []
modify_flag = []
# return mongodb report
for nodule_report in userdb.report.find({
'uid': request.uid,
'scanID': request.scanID
}):
modify_flag.append(True)
#nodule index
nindex = nodule_report['nodule']
# status
list_ds.append(thyroidrpc_pb2.DetectionStatus(n=nindex, status=0))
# position
x, y, w, h, = nodule_report['pos']
s = nodule_report['cut']
pos_extend = nodule_report['pos_extend']
list_nodule.append(
thyroidrpc_pb2.NoduleWithNum(n=nindex,
m=0,
x=x,
y=y,
w=w,
h=h,
s=s,
pos=pos_extend))
# classification
benign = nodule_report['benign']
prob = nodule_report['prob']
list_benign_prob.append(
thyroidrpc_pb2.BenignAndProb(benign=benign, prob=prob))
# tirads
constitute = nodule_report['constitute']
comet = nodule_report['comet']
shape = nodule_report['shape']
border_clear = nodule_report['border_clear']
echo_level = nodule_report['echo_level']
ratio = nodule_report['ratio']
hxlen = nodule_report['hxlen']
vxlen = nodule_report['vxlen']
calcification = nodule_report['calcification']
list_tirads.append(
thyroidrpc_pb2.OneTiradsRes(constitute=constitute,
comet=comet,
shape=shape,
ratio=ratio,
hxlen=hxlen,
vxlen=vxlen,
echo_level=echo_level,
border_clear=border_clear,
calcification=calcification))
nums = userdb.report.count_documents({
'uid': request.uid,
'scanID': request.scanID
})
anypb = Any()
anypb.Pack(
thyroidrpc_pb2.CTResponse(nums=nums,
ds=list_ds,
nodule=list_nodule,
bp=list_benign_prob,
tirads=list_tirads,
modify_flag=modify_flag))
t2 = time.perf_counter()
logging.info('read report: {:.2f}'.format((t2 - t1) * 1000))
return thyroidrpc_pb2.ProtoResponse(code=0, msg='', data=anypb)
def ModifyReport(self, request, context):
""" modify report, only support modify one nodule per call!
"""
"""
'benign': benign,
'prob': prob,
'shape': shape,
'border_clear': border_clear,
'constitute': constitute,
'echo_level': echo_level,
'cut': 0, # just a placeholder, meaningless
'ratio': ratio,
'hxlen': hxlen,
'vxlen': vxlen,
'comet': comet,
'calcification': calcification
"""
logging.info('---------------------------------------')
logging.info('modifing report')
# 1. update benign
if len(request.benign) == 1:
newval = request.benign[0]
userdb.report.find_one_and_update(
{
'uid': request.uid,
'scanID': request.scanID,
'nodule': request.nindex
}, {'$set': {
'flag_modified': True,
'benign': newval
}})
# 2. update prob
if len(request.prob) == 1:
newval = request.prob[0]
userdb.report.find_one_and_update(
{
'uid': request.uid,
'scanID': request.scanID,
'nodule': request.nindex
}, {'$set': {
'flag_modified': True,
'prob': newval
}})
# 3. update constitute
if len(request.constitute) == 1:
newval = request.constitute[0]
userdb.report.find_one_and_update(
{
'uid': request.uid,
'scanID': request.scanID,
'nodule': request.nindex
}, {'$set': {
'flag_modified': True,
'constitute': newval
}})
# 4. update comet
if len(request.comet) == 1:
newval = request.comet[0]
userdb.report.find_one_and_update(
{
'uid': request.uid,
'scanID': request.scanID,
'nodule': request.nindex
}, {'$set': {
'flag_modified': True,
'comet': newval
}})
# 5. update shape
if len(request.shape) == 1:
newval = request.shape[0]
userdb.report.find_one_and_update(
{
'uid': request.uid,
'scanID': request.scanID,
'nodule': request.nindex
}, {'$set': {
'flag_modified': True,
'shape': newval
}})
# 6. update echo_level
if len(request.echo_level) == 1:
newval = request.echo_level[0]
doc = userdb.report.find_one({
'uid': request.uid,
'scanID': request.scanID,
'nodule': request.nindex
})
logging.info('modify-before')
logging.info(doc)
userdb.report.find_one_and_update(
{
'uid': request.uid,
'scanID': request.scanID,
'nodule': request.nindex
}, {'$set': {
'flag_modified': True,
'echo_level': newval
}})
doc = userdb.report.find_one({
'uid': request.uid,
'scanID': request.scanID,
'nodule': request.nindex
})
logging.info('modify-after')
logging.info(doc)
# 7. update border_clear
if len(request.border_clear) == 1:
newval = request.border_clear[0]
userdb.report.find_one_and_update(
{
'uid': request.uid,
'scanID': request.scanID,
'nodule': request.nindex
}, {'$set': {
'flag_modified': True,
'border_clear': newval
}})
# 8. update calcification
if len(request.calcification) == 3:
newval = request.calcification
userdb.report.find_one_and_update(
{
'uid': request.uid,
'scanID': request.scanID,
'nodule': nindex
}, {'$set': {
'flag_modified': True,
'calcification': newval
}})
return thyroidrpc_pb2.ProtoResponse(code=0, msg='', data=Any())
def GenerateReport(self, request, context):
"""generating report function
"""
logging.info('---------------------------------------')
logging.info('GenerateReport')
t1 = time.perf_counter()
logging.info(f'uid: {request.uid}, scanID: {request.scanID}')
# we should only process the un-generate report records
# record has flag *flag_report*
# at first count module appearing times in all images
nodule_dict = {} # key(nodule index): value(appear times)
for user in userdb.collection.find({
'uid': request.uid,
'scanID': request.scanID,
'flag_report': False
}):
key = user['nodule']
nodule_dict[key] = nodule_dict.get(key, 0) + 1
# merge results to one record
# loop over nodule index
for index, value in nodule_dict.items():
logging.info(f'nodule index: {index}, un-gerated records: {value}')
if value > 1:
# merge results
one_record = self._merge_records(request.uid, request.scanID,
index, value)
else:
# set *flag_report* True, means this record has been used in final report!
one_record = userdb.collection.find_one_and_update(
{
'uid': request.uid,
'scanID': request.scanID,
'nodule': index
}, {'$set': {
'flag_report': True
}},
return_document=pymongo.ReturnDocument.AFTER)
del one_record['_id']
del one_record['imageID']
del one_record['flag_report']
one_record[
'flag_modified'] = False # means the doctor does not modify the report!
# merge one_record and report
report = userdb.report.find_one({
'uid': request.uid,
'scanID': request.scanID,
'nodule': index
})
# if no report, just insert one!
if report is None:
userdb.report.insert_one(one_record)
else:
self._merge_record_and_report(one_record, report)
# output list
list_ds = []
list_nodule = []
list_benign_prob = []
list_tirads = []
modify_flag = []
# return mongodb report
for nodule_report in userdb.report.find({
'uid': request.uid,
'scanID': request.scanID
}):
# modify_flag
modify_flag.append(False)
#nodule index
nindex = nodule_report['nodule']
# status
list_ds.append(thyroidrpc_pb2.DetectionStatus(n=nindex, status=0))
# position
x, y, w, h, = nodule_report['pos']
s = nodule_report['cut']
pos_extend = nodule_report['pos_extend']
list_nodule.append(
thyroidrpc_pb2.NoduleWithNum(n=nindex,
m=0,
x=x,
y=y,
w=w,
h=h,
s=s,
pos=pos_extend))
# classification
benign = nodule_report['benign']
prob = nodule_report['prob']
list_benign_prob.append(
thyroidrpc_pb2.BenignAndProb(benign=benign, prob=prob))
# tirads
constitute = nodule_report['constitute']
comet = nodule_report['comet']
shape = nodule_report['shape']
border_clear = nodule_report['border_clear']
echo_level = nodule_report['echo_level']
ratio = nodule_report['ratio']
hxlen = nodule_report['hxlen']
vxlen = nodule_report['vxlen']
calcification = nodule_report['calcification']
list_tirads.append(
thyroidrpc_pb2.OneTiradsRes(constitute=constitute,
comet=comet,
shape=shape,
ratio=ratio,
hxlen=hxlen,
vxlen=vxlen,
echo_level=echo_level,
border_clear=border_clear,
calcification=calcification))
nums = userdb.report.count_documents({
'uid': request.uid,
'scanID': request.scanID
})
anypb = Any()
anypb.Pack(
thyroidrpc_pb2.CTResponse(nums=nums,
ds=list_ds,
nodule=list_nodule,
bp=list_benign_prob,
tirads=list_tirads,
modify_flag=modify_flag))
t2 = time.perf_counter()
logging.info('generate report: {:.2f}'.format((t2 - t1) * 1000))
return thyroidrpc_pb2.ProtoResponse(code=0, msg='', data=anypb)
def ClassifyAndTirads(self, request, context):
""" classification, tirads
"""
logging.info('---------------------------------------')
logging.info('ClassifyAndTirads')
logging.info(
f'uid: {request.uid}, scanID: {request.scanID}, imageID: {request.imageID}'
)
anypb = Any()
image_gray = np.frombuffer(request.image, dtype=np.uint8)
height = request.height
width = request.width
if not request.isRaw:
image_gray = cv2.imdecode(image_gray, cv2.IMREAD_GRAYSCALE)
height, width = image_gray.shape
# check image size
if 0 == image_gray.size or image_gray.size != height * width:
logging.error(
f'image shape is {image_gray.shape}, but height = {height}, width = {width}'
)
return thyroidrpc_pb2.ProtoResponse(
code=WRONG_IMAGE_SHAPE,
msg=ErrorDict[WRONG_IMAGE_SHAPE],
data=anypb)
if image_gray.ndim == 1:
image_gray = image_gray.reshape([height, width])
logging.info(f'image shape is height = {height}, width = {width}')
# gray to rgb
image = cv2.cvtColor(image_gray, cv2.COLOR_GRAY2BGR)
# check input nodule number
if len(request.nodule) <= 0:
logging.error(
f'input nodule numbers should large than zero, but your input is: {len(request.nodule)}'
)
return thyroidrpc_pb2.ProtoResponse(
code=WRONG_INPUT_NODELES,
msg=ErrorDict[WRONG_INPUT_NODELES],
data=anypb)
"""
what's the logic?
1. loop over request bounding box, use second stage maskrcnn to generate new mask
2. for each new mask/nodule, do classification task
3. for each new mask/nodule, do tirads task
4. save the results for final report
"""
# output list
list_ds = []
list_nodule = []
list_benign_prob = []
list_tirads = []
modify_flag = []
# loop over all the nodules
for nodule in request.nodule:
modify_flag.append(False)
t1 = time.perf_counter()
logging.info(
f'nodule index: {nodule.n}, pos = ({nodule.x}, {nodule.y}, {nodule.w}, {nodule.h})'
)
# 1. do second stage segmentation
bbox_input = (nodule.x, nodule.y, nodule.x + nodule.w,
nodule.y + nodule.h)
try:
mask, bbox = maskrcnn2seg.do_inference(image_gray, bbox_input)
print(bbox)
except:
logging.error('second stage detection exception')
return thyroidrpc_pb2.ProtoResponse(
code=SECOND_DETECTION_FAIL,
msg=ErrorDict[SECOND_DETECTION_FAIL],
data=anypb)
if mask.size == 0:
logging.error(
f'{nodule.n}: second stage not detect any nodule!')
list_ds.append(
thyroidrpc_pb2.DetectionStatus(n=nodule.n, status=1))
list_nodule.append([])
list_benign_prob.append([])
list_tirads.append([])
continue
else:
list_nodule.append(
thyroidrpc_pb2.NoduleWithNum(n=nodule.n,
m=nodule.m,
x=bbox[0],
y=bbox[1],
w=bbox[2] - bbox[0],
h=bbox[3] - bbox[1],
s=nodule.s,
pos=nodule.pos))
bbox_pos = [
bbox[0].item(), bbox[1].item(), (bbox[2] - bbox[0]).item(),
(bbox[3] - bbox[1]).item()
]
t2 = time.perf_counter()
logging.info('nodule {}, second detection time: {:.2f}'.format(
nodule.n, (t2 - t1) * 1000))
# 2. do classification
t1 = time.perf_counter()
try:
benign, prob = rescnn.do_inference(image_gray, mask)
print(benign, prob)
except:
logging.error('classification fail!')
return thyroidrpc_pb2.ProtoResponse(
code=CLASSIFICATION_FAIL,
msg=ErrorDict[CLASSIFICATION_FAIL],
data=anypb)
# 1: benign; 2: not benign
benign += 1
list_benign_prob.append(
thyroidrpc_pb2.BenignAndProb(benign=benign, prob=prob))
t2 = time.perf_counter()
logging.info('nodule {}, classification time: {:.2f}'.format(
nodule.n, (t2 - t1) * 1000))
# 3. do tirads
"""
constitute 构成 0： 实性 1： 实性为主 2： 囊性为主 3： 囊性
comet 彗星尾数量， 不过不要显示数量，直接大于1时显示有彗星尾就行
shape 0： 形状规则 1： 形状不规则
ar # 纵横比
hx_len # 水平轴 如果是横切，它就是左右径，如果是纵切，它就是上下径
vx_len # 垂直轴 它只有可能是前后径
echo_code # 回声水平 0： 无回声 2： 低回声 3： 等回声 4： 高回声
border_clear # 边界清晰模糊 0： 模糊 1： 清晰
calcification[0] # 0：无微钙化 1： 有微钙化
calcification[1] # 0：无粗钙化 1： 有粗钙化
calcification[2] # 0：无环钙化 1： 有环钙化
"""
# 3.0 diffuse, c_s
try:
anno, c_s = pymrcnn_part.ThyroidAIDoInference(
image_gray, height, width)
print(anno, c_s)
except:
logging.error('part detection fail!')
return thyroidrpc_pb2.ProtoResponse(
code=TIRADS_FAIL,
msg=ErrorDict[CLASSIFICATION_FAIL],
data=anypb)
try:
if nodule.w * nodule.h < 100000:
diffusion = diffuse.inference(image_gray, anno)
else:
diffusion = 2
except:
logging.error('diffuse fail!')
return thyroidrpc_pb2.ProtoResponse(
code=TIRADS_FAIL,
msg=ErrorDict[CLASSIFICATION_FAIL],
data=anypb)
t1 = time.perf_counter()
tirads = TiradsRecognition(image_gray, mask, is_debug=False)
# 3.1 constitute
constitute, cys_mask = tirads.find_or_report_constitute("report")
# 3.2 comet
if constitute <= 1:
comet = 0
else:
comet = tirads.find_comet_calcify("report", cys_mask)
# 3.3 ratio and hx_len, vx_len
ar, vx, hx = tirads.estimate_aspect_ratio()
hx_len, vx_len = tirads.get_axis_len(hx), tirads.get_axis_len(vx)
pixels_one_cm = request.ppc
hx_len /= pixels_one_cm
vx_len /= pixels_one_cm
hx_len = round(hx_len, 8)
vx_len = round(vx_len, 8)
if nodule.s == 1:
hx_list = [hx_len, 0]
else:
hx_list = [0, hx_len]
# 3.4 shape
shape = tirads.classify_shape()
# TODO: echo lever
# 3.5 echo level
#echo_code = tirads.compute_nodule_echo(mask_cyst=cys_mask)
echo_code = 2
# 3.6 border
border_clear = tirads.border_clear_fuzzy()
# 3.7 calcification 钙化
if comet > 0:
calcification = tirads.find_or_report_calcification(
pixels_one_cm)
else:
calcification = [0, 0, 0]
# 加入弥漫性结果
calcification.append(diffusion)
list_tirads.append(
thyroidrpc_pb2.OneTiradsRes(constitute=constitute,
comet=comet,
shape=shape,
ratio=ar,
hxlen=hx_list,
vxlen=vx_len,
echo_level=echo_code,
border_clear=border_clear,
calcification=calcification))
# at last we set status ok
list_ds.append(thyroidrpc_pb2.DetectionStatus(n=nodule.n,
status=0))
t2 = time.perf_counter()
logging.info('nodule {}, tirads time: {:.2f}'.format(
nodule.n, (t2 - t1) * 1000))
# save results to mongodb
# if need update, replace old result
if nodule.m > 0:
old_index = nodule.m
logging.info(f'rename nodule index: {nodule.m} -> {nodule.n}')
else:
old_index = nodule.n
logging.info(f'and/replace nodule: {nodule.n}')
try:
userdb.collection.find_one_and_replace(
{
'uid': request.uid,
'scanID': request.scanID,
'imageID': request.imageID,
'nodule': old_index
},
{
'uid': request.uid,
'scanID': request.scanID,
'imageID': request.imageID,
'nodule': nodule.n,
'flag_report': False, # means not used by report!
'pos': bbox_pos,
'pos_extend': nodule.pos,
'benign': benign,
'prob': prob,
'comet': comet,
'constitute': constitute,
'shape': shape,
'echo_level': echo_code,
'border_clear': border_clear,
'cut': nodule.s,
'ratio': ar,
'hxlen': hx_list,
'vxlen': vx_len,
'calcification': calcification
},
upsert=True # if not find, just insert
)
except:
logging.error('userdb.collection.find_one_and_replace fail!')
return thyroidrpc_pb2.ProtoResponse(
code=DB_REPLACE_FAIL,
msg=ErrorDict[DB_REPLACE_FAIL],
data=anypb)
# pack the proto result
anypb.Pack(
thyroidrpc_pb2.CTResponse(nums=len(request.nodule),
ds=list_ds,
nodule=list_nodule,
bp=list_benign_prob,
tirads=list_tirads,
modify_flag=modify_flag))
return thyroidrpc_pb2.ProtoResponse(code=0, msg='', data=anypb)