def fill_segments(segments, device_type):
values = []
colors = []
fields = get_fields_info([device_type])
for s in segments:
values.append({"name": s['name'], "value": get_field_rand_value(fields[s['name']])})
colors.append((random.randint(20,255),random.randint(20,255),random.randint(20,255)))
return values, colors
def test_is_text_valid_float():
devices = get_fields_info()
assert is_text_valid(add_decimal_notation("37.8", devices["Temp"]),
devices["Temp"])
assert is_text_valid(add_decimal_notation("378", devices["Temp"]),
devices["Temp"])
assert is_text_valid(add_decimal_notation("370", devices["Temp"]),
devices["Temp"])
def test_ocr_by_segments():
image = imageio.imread(
os.path.dirname(__file__) +
"/data/cvmonitors-cvmonitor-9a6d4db29fa04bfa.jpg")
segments = json.load(
open(
os.path.dirname(__file__) +
"/data/cvmonitors-cvmonitor-9a6d4db29fa04bfa.json"))
devices = get_fields_info()
text_spotting = ocr.text_spotting.text_spotting.Model()
expected_boxes = get_ocr_expected_boxes(segments["segments"], devices, 0.5,
0.6)
texts, boxes, scores, _ = text_spotting.forward(
image, expected_boxes=expected_boxes)
def create_segments(device_type, fontScale, thickness, image_size=[1000, 1200], x_start=300, y_start=200):
size=np.array(cv2.getTextSize(text=str('COFFE'), fontFace=cv2.FONT_HERSHEY_PLAIN, fontScale=fontScale,thickness=thickness)[0])+10
#size=np.array(cv2.getTextSize(text=str('CO'), fontFace=cv2.FONT_HERSHEY_PLAIN, fontScale=fontScale,thickness=thickness)[0])+10
y_step = size[1]+50
x_step = size[0]+50
x = x_start
y = y_start
segments = []
fields = get_fields_info([device_type])
for m in fields:
if x+x_step >= image_size[1]-50:
x = x_start
y += y_step
segments.append({
"top": max(y-20,0),
"left": max(x-20,0),
"bottom": int(min(y+size[1],image_size[0]-10)),
"right": int(min(x+size[0],image_size[1]-10)),
"name": m,
})
if x+x_step < image_size[1]-50:
x += x_step
return segments
def change_values(values, device_type):
fields = get_fields_info([device_type])
for i in range(len(values)):
values[i]['value']= get_field_rand_value(fields[values[i]['name']], values[i]['value'])
return values
"עמיר פרץ",
"ניצן הורוביץ",
"תמר זנדברג",
"איציק שמולי",
"מרב מיכאלי",
"יאיר גול",
"נפתלי בנט",
"רפי פרץ",
"איילת שקד",
"בצלאל סמוטריץ'",
"מתן כהנא",
"אופיר סופ",
"אורלי לוי-אבקסיס"
]
devices = get_fields_info()
def get_qr_code(title):
qr = qrcode.QRCode(error_correction=qrcode.constants.ERROR_CORRECT_L)
# Seedable uuid4:
a = "%32x" % random.getrandbits(128)
rd = a[:12] + '4' + a[13:16] + 'a' + a[17:]
uuid4 = uuid.UUID(rd)
text = f'cvmonitors-{title}-{uuid4.hex[:16]}'
qr.add_data(text)
qr.make(fit=True)
img = qr.make_image(fill_color='black', back_color='white')
qrsize = random.randint(120,180)
img = cv2.resize(np.array(img,dtype=np.uint8)*255,(qrsize,qrsize))
img= cv2.cvtColor(img,cv2.COLOR_GRAY2BGR)
def test_is_text_valid_int():
devices = get_fields_info()
assert is_text_valid("100", devices["HR"])
def test_is_text_valid_str():
devices = get_fields_info()
assert is_text_valid("simv+", devices["Medication Name"])
def __init__(self):
self.blueprint = Blueprint("cv", __name__)
self.qrDecoder = cv2.QRCodeDetector()
self.model_ocr = None
self.devices = get_fields_info()
self.resultsLogger = ResultLogger()
prob_threshold = float(
os.environ.get("CVMONITOR_SPOTTING_PROB_THRESHOLD", "0.3"))
max_seq_len = int(
os.environ.get("CVMONITOR_SPOTTING_MAX_SEQ_LEN", "10"))
iou_threshold = float(
os.environ.get("CVMONITOR_SPOTTING_IOU_THRESHOLD", "0.01"))
sportting_model = os.environ.get("CVMONITOR_SPOTTING_MODEL_TYPE",
"FP32")
self.text_spotting = text_spotting.Model(
prob_threshold=prob_threshold,
max_seq_len=max_seq_len,
iou_threshold=iou_threshold,
model_type=sportting_model,
)
@self.blueprint.route("/ping/")
def ping():
return "pong cv"
@self.blueprint.route("/detect_codes", methods=["POST"])
def detect_codes():
"""
Get QR or barcodes in an image
---
description: Get QR codes and barcodes in an image
requestBody:
content:
image/png:
schema:
type: string
format: binary
responses:
'200':
dsecription: array of detections
content:
application/json:
schema:
type: array
items:
type: object
properties:
data:
type: string
top:
type: number
left:
type: number
bottom:
type: number
right:
type: number
"""
image = np.asarray(imageio.imread(request.data))
codes = read_codes(image)
return json.dumps(codes), 200, {"content-type": "application/json"}
@self.blueprint.route("/align_image", methods=["POST"])
def align_image():
"""
Given a jpeg image with that containes the QR code, use that QR code to align the image
---
description: Gets a jpeg and returns a jpeg
requestBody:
content:
image/png:
schema:
type: string
format: binary
responses:
'200':
descritption: jpeg image
content:
image/png:
schema:
type: string
format: binary
"""
corners = []
try:
corners = json.loads(request.headers.get('X-CORNERS'))
except:
pass
use_exif = os.environ.get("CVMONITOR_ORIENT_BY_EXIF",
"TRUE") == "TRUE"
use_qr = os.environ.get("CVMONITOR_ORIENT_BY_QR",
"FALSE") == "TRUE"
qrprefix = str(os.environ.get("CVMONITOR_QR_PREFIX", "cvmonitor"))
qrsize = int(os.environ.get("CVMONITOR_QR_TARGET_SIZE", 100))
boundery = float(os.environ.get("CVMONITOR_QR_BOUNDERY_SIZE", 50))
align_image_by_qr = (os.environ.get("CVMONITOR_SKIP_ALIGN",
"TRUE") == "FALSE")
save_before_align = os.environ.get(
"CVMONITOR_SAVE_BEFORE_ALIGN") == "TRUE"
save_after_align = os.environ.get(
"CVMONITOR_SAVE_AFTER_ALIGN") == "TRUE"
imdata = io.BytesIO(request.data)
image, detected_qrcode, _ = get_oriented_image(imdata,
use_exif=use_exif,
use_qr=use_qr)
headers = {"content-type": "image/jpeg"}
if save_before_align:
imdata.seek(0)
with open("original_image.jpg", "wb") as f:
f.write(imdata)
if detected_qrcode is None:
if align_image_by_qr:
abort(
400,
"Could not align the image by qr code, no such code detected",
)
else:
headers["X-MONITOR-ID"] = detected_qrcode.data.decode()
if align_image_by_qr:
logging.debug("Trying to align image by qr code")
image, _ = align_by_qrcode(image, detected_qrcode, qrsize,
boundery, qrprefix)
if save_after_align:
imageio.imwrite("aligned_image.jpg", image)
b = io.BytesIO()
imageio.imwrite(b, image, format="jpeg")
b.seek(0)
return b.read(), 200, headers
@self.blueprint.route("/run_ocr", methods=["POST"])
def run_ocr():
"""
Run ocr on an image
---
description: run ocr on image
requestBody:
content:
application/json:
schema:
type: object
properties:
image:
type: string
contentEncoding: base64
contentMediaType: image/jpeg
segments:
type: array
items:
type: object
properties:
top:
type: number
format: integer
left:
type: number
format: integer
bottom:
type: number
format: integer
right:
type: number
format: integer
responses:
'200':
description: ocr results
content:
application/json:
schema:
type: array
items:
type: object
properties:
segment_name:
type: string
value:
type: string
"""
segment_threshold = float(
os.environ.get("CVMONITOR_SEGMENT_THRESHOLD", "0.95"))
device_ocr_default_score = float(
os.environ.get("CVMONITOR_DEVICE_OCR_DEFAULT_SCORE", "0.5"))
device_ocr_score_threshold = float(
os.environ.get("CVMONITOR_DEVICE_OCR_SCORE_THRESHOLD", "0.8"))
suggest_segments = float(
os.environ.get("CVMONITOR_SUGGEST_SEGMENT", "FALSE") == "TRUE")
spotting_ocr = os.environ.get("CVMONITOR_OCR_SPOTTING",
"TRUE") == "TRUE"
threshold = float(
os.environ.get("CVMONITOR_SERVER_OCR_THRESHOLD", "0.8"))
if not self.model_ocr:
self.model_ocr = monitor_ocr.build_model()
if not self.model_ocr:
abort(500, "NN Model not found, could not run ocr")
data = request.json
assert "image" in data
image_data = base64.decodebytes(data["image"].encode())
image = np.asarray(imageio.imread(image_data))
# Suggest segments
have_names = False
for s in data.get("segments", []) or []:
if 'name' in s:
have_names = True
if suggest_segments and (not data.get("segments")
or not have_names):
# Let's run segment detection.
texts, boxes, scores, _ = self.text_spotting.forward(image)
segments = []
for text, box, score in zip(texts, boxes, scores):
if score > segment_threshold:
segments.append({
"value": text,
"left": float(box[0]),
"top": float(box[1]),
"right": float(box[2]),
"bottom": float(box[3]),
"score": float(score),
"source": "server",
})
logging.debug(f"Detections (new): {segments}")
return json.dumps(segments), 200, {
"content-type": "application/json"
}
segments = copy.deepcopy(data.get("segments", []))
if len(segments) == 0:
logging.error("No segments")
return json.dumps([]), 200, {
"content-type": "application/json"
}
# We will at most give results on segments:
expected_boxes = get_ocr_expected_boxes(
segments,
self.devices,
device_ocr_default_score,
device_ocr_score_threshold,
)
if spotting_ocr:
texts, boxes, scores, _ = self.text_spotting.forward(
image, expected_boxes=expected_boxes)
else:
texts, scores = self.model_ocr.ocr(expected_boxes, image,
threshold)
should_log = False
if texts:
for eb, text, score in zip(expected_boxes, texts, scores):
if score > threshold:
if segments[eb["index"]].get("value") != text:
should_log = True
segments[eb["index"]]["value"] = text
segments[eb["index"]]["score"] = float(score)
segments[eb["index"]]["source"] = "server"
if should_log or random.randint(0, 100) == 0 or not texts:
self.resultsLogger.log_ocr(
image_data, data["segments"], {
'expected': expected_boxes,
'texts': texts,
'scores': [float(s) for s in scores]
})
for s in segments:
if 'value' not in s:
s['value'] = None
logging.debug(f"Detections: {segments}")
return json.dumps(segments), 200, {
"content-type": "application/json"
}
@self.blueprint.route("/show_ocr/", methods=["POST"])
def show_ocr():
"""
Run ocr on an image
---
description: run ocr on image
requestBody:
content:
application/json:
schema:
type: object
properties:
image:
type: string
contentEncoding: base64
contentMediaType: image/jpeg
segments:
type: array
items:
type: object
properties:
top:
type: number
format: integer
left:
type: number
format: integer
bottom:
type: number
format: integer
right:
type: number
format: integer
responses:
'200':
description: ocr results
content:
application/json:
schema:
type: array
items:
type: object
properties:
segment_name:
type: string
value:
type: string
"""
data = request.json
assert "image" in data
image = np.asarray(
imageio.imread(base64.decodebytes(data["image"].encode())))
# Suggest segments
if data.get("segments"):
image = draw_segments(image, data.get('segments'))
headers = {"content-type": "image/jpeg"}
b = io.BytesIO()
imageio.imwrite(b, image, format="jpeg")
b.seek(0)
return b.read(), 200, headers
@self.blueprint.route("/qr/<title>", methods=["GET"])
def qr(title):
"""
Generate pdf of qr codes, after the /qr/ put title for
each qr.
The data in the qr code will be cvmonitor-title-16_random_characters
---
description: get pdf of qr codes
get:
parameters:
- in: path
name: title
schema:
type: string
required: true
default: cvmonitor
- in: query
name: width
schema:
type: number
required: false
- in: query
name: height
schema:
type: number
required: false
responses:
'200':
descritption: pdf of results
content:
application/pdf:
"""
try:
width = int(request.args.get("width"))
except:
width = None
try:
height = int(request.args.get("height"))
except:
height = None
headers = {
"Content-Type": "application/pdf",
"Content-Disposition": 'attachment; filename="random-qr.pdf"',
}
pdf_buffer = io.BytesIO()
generate_pdf(pdf_buffer, title, width, height)
pdf_buffer.seek(0)
return pdf_buffer.read(), 200, headers
@self.blueprint.route("/measurements/<device>", methods=["GET"])
def get_measurements(device):
return json.dumps([x for x in get_fields_info([device]).keys()
]), 200, {
'content-type': 'application/json'
}
def get_measurements(device):
return json.dumps([x for x in get_fields_info([device]).keys()
]), 200, {
'content-type': 'application/json'
}
def __init__(self,
device='CPU',
track=False,
visualize=False,
prob_threshold=0.3,
max_seq_len=10,
iou_threshold=0.4,
model_type='FP32',
rgb2bgr=True):
assert (model_type == 'FP32') or (model_type == 'FP16')
mask_rcnn_model_xml = get_models()[
'{}/text-spotting-0001-detector.xml'.format(model_type)]
mask_rcnn_model_bin = get_models()[
'{}/text-spotting-0001-detector.bin'.format(model_type)]
text_enc_model_xml = get_models()[
'{}/text-spotting-0001-recognizer-encoder.xml'.format(model_type)]
text_enc_model_bin = get_models()[
'{}/text-spotting-0001-recognizer-encoder.bin'.format(model_type)]
text_dec_model_xml = get_models()[
'{}/text-spotting-0001-recognizer-decoder.xml'.format(model_type)]
text_dec_model_bin = get_models()[
'{}/text-spotting-0001-recognizer-decoder.bin'.format(model_type)]
# Plugin initialization for specified device and load extensions library if specified.
log.info('Creating Inference Engine...')
ie = IECore()
# Read IR
log.info('Loading network files:\n\t{}\n\t{}'.format(
mask_rcnn_model_xml, mask_rcnn_model_bin))
mask_rcnn_net = IENetwork(model=mask_rcnn_model_xml,
weights=mask_rcnn_model_bin)
log.info('Loading network files:\n\t{}\n\t{}'.format(
text_enc_model_xml, text_enc_model_bin))
text_enc_net = IENetwork(model=text_enc_model_xml,
weights=text_enc_model_bin)
log.info('Loading network files:\n\t{}\n\t{}'.format(
text_dec_model_xml, text_dec_model_bin))
text_dec_net = IENetwork(model=text_dec_model_xml,
weights=text_dec_model_bin)
supported_layers = ie.query_network(mask_rcnn_net, 'CPU')
not_supported_layers = [
l for l in mask_rcnn_net.layers.keys() if l not in supported_layers
]
if len(not_supported_layers) != 0:
log.error(
'Following layers are not supported by the plugin for specified device {}:\n {}'
.format(device, ', '.join(not_supported_layers)))
log.error(
"Please try to specify cpu extensions library path in sample's command line parameters using -l "
"or --cpu_extension command line argument")
sys.exit(1)
required_input_keys = {'im_data', 'im_info'}
assert required_input_keys == set(mask_rcnn_net.inputs.keys()), \
'Demo supports only topologies with the following input keys: {}'.format(', '.join(required_input_keys))
required_output_keys = {
'boxes', 'scores', 'classes', 'raw_masks', 'text_features'
}
assert required_output_keys.issubset(mask_rcnn_net.outputs.keys()), \
'Demo supports only topologies with the following output keys: {}'.format(', '.join(required_output_keys))
n, c, h, w = mask_rcnn_net.inputs['im_data'].shape
assert n == 1, 'Only batch 1 is supported by the demo application'
self.shape = [n, c, h, w]
log.info('Loading IR to the plugin...')
self.mask_rcnn_exec_net = ie.load_network(network=mask_rcnn_net,
device_name=device,
num_requests=2)
self.text_enc_exec_net = ie.load_network(network=text_enc_net,
device_name=device)
self.text_dec_exec_net = ie.load_network(network=text_dec_net,
device_name=device)
self.hidden_shape = text_dec_net.inputs['prev_hidden'].shape
self.prob_threshold = prob_threshold
self.tracker = None
self.visualizer = None
self.iou_threshold = iou_threshold
self.max_seq_len = max_seq_len
self.rgb2bgr = rgb2bgr
self.device_names = get_fields_info()
if track:
self.tracker = StaticIOUTracker()
if visualize:
self.visualizer = Visualizer(['__background__', 'text'],
show_boxes=True,
show_scores=True)
log.info('Model ready...')