def test_threaded_streamer(self): streamer = ThreadedStreamer(self.vision_model.batch_prediction, batch_size=8) single_predict = streamer.predict(self.input_batch) assert single_predict == self.single_output batch_predict = streamer.predict(self.input_batch * BATCH_SIZE) assert batch_predict == self.batch_output
def test_threaded_streamer(): streamer = ThreadedStreamer(vision_model.batch_prediction, batch_size=16) output_raw = vision_model.batch_prediction(input_batch) output = streamer.predict(input_batch) assert output_raw == output outputs_raw = vision_model.batch_prediction(input_batch * 55) outputs = streamer.predict(input_batch * 55) assert outputs_raw == outputs
def test_future_api(self): streamer = ThreadedStreamer(self.vision_model.batch_prediction, batch_size=8) xs = [] for i in range(BATCH_SIZE): future = streamer.submit(self.input_batch) xs.append(future) batch_predict = [] # Get all instances of future object and wait for asynchronous responses. for future in xs: batch_predict.extend(future.result()) assert batch_predict == self.batch_output
from service_streamer import ThreadedStreamer app = Flask(__name__) model = None streamer = None @app.route("/naive", methods=["POST"]) def naive_predict(): inputs = request.form.getlist("s") outputs = model.predict(inputs) return jsonify(outputs) @app.route("/stream", methods=["POST"]) def stream_predict(): inputs = request.form.getlist("s") outputs = streamer.predict(inputs) return jsonify(outputs) if __name__ == "__main__": model = Model() # start child thread as worker streamer = ThreadedStreamer(model.predict, batch_size=64, max_latency=0.1) # spawn child process as worker # streamer = Streamer(model.predict, batch_size=64, max_latency=0.1) app.run(port=5005, debug=False)
return jsonify({'result': outputs[0][0].tolist()}) def batch_prediction(image_bytes_batch): image_tensors = [ preprocess_image(image, target_size=(416, 416)) for image in image_bytes_batch ] tensor = np.concatenate(image_tensors, axis=0) outputs = face_mask.forward([tensor]) return [outputs[0][i] for i in range(len(outputs[0]))] streamer = ThreadedStreamer(batch_prediction, batch_size=16) @app.route('/stream_predict', methods=['POST']) def stream_predict(): message = request.get_json(force=True) encoded = message['image'] decoded = base64.b64decode(encoded) image = Image.open(io.BytesIO(decoded)) results = streamer.predict([image])[0] return jsonify({'result': results.tolist()}) @app.route('/', methods=['GET']) def index():
try: with open("config/jaeger-config.json", 'r') as f: jaeger_config_json = json.load(f) except Exception as e: logger.error(e) exit() port = int(config_json["translate-service"]["port"]) model_dir = config_json["translate_model_dir"] logger.info("port:{}".format(port)) logger.info("model:{}".format(model_dir)) #model_dir = "/home/wzhang/lyx/DeathStarBench/socialNetwork/data/ende_ctranslate2" translate_func = TransFunc() jaeger_config = Config(config=jaeger_config_json, service_name="translate-service", validate=True) tracer = jaeger_config.initialize_tracer() tokenizer = pyonmttok.Tokenizer('conservative', joiner_annotate=True) streamer = ThreadedStreamer(translate_func.predict, batch_size=batch_size, max_latency=wait_time) handler = TranslateHandler(tokenizer = tokenizer, streamer=streamer, tracer=tracer) processor = TranslateService.Processor(handler) transport = TSocket.TServerSocket("0.0.0.0", port) tfactory = TTransport.TFramedTransportFactory() pfactory = TBinaryProtocol.TBinaryProtocolFactory() server = TServer.TThreadedServer( processor, transport, tfactory, pfactory) logger.info("Starting the translate-service server...") server.serve() logger.info("translate-service server exit...")
return [get_preds(outp)] img_transforms = get_transforms imagenete_id_cat = json.load(open('imagenet_class_index.json')) def get_img(url): response = requests.get(url) img = Image.open(BytesIO(response.content)) return img def get_preds(preds): idx = preds.max(1)[1].item() return imagenete_id_cat[str(idx)][1] @app.route('/predict',methods=['POST']) def predict(): img = get_img(request.args['url']) img_t = img_transforms()(img).unsqueeze(0).to(device) outp = model(img_t) return f'{get_preds(outp)}' streamer = ThreadedStreamer(predict_batch,batch_size=64,max_latency=0.1) @app.route("/stream", methods=["POST"]) def stream_predict(): url = request.args['url'] outp = streamer.predict([url])[0] return str(outp) if __name__ == '__main__': app.run(debug=False)
if type(img) is list: img = img[0] result = inference_segmentor(model, img) return result def get_random_img(n=1): img_table_file = base + 'img_anno.csv' # test_dir = 'output/ocr_parkinglot/' img_dir = base + 'images/' # anno_dir = base+'labels/' img_table = pd.read_csv(img_table_file) paths = img_table.dropna().sample(n).img.to_list() # img_name = path.split('/')[-1] # img_path = img_dir+img_name return paths ## Local test # paths = get_random_img() # img_path = paths[0] # result = predict_save(model, img_path) # polygons = polygonize(result, draw_img=img_path) if __name__ == "__main__": # start child thread as worker streamer = ThreadedStreamer(predict, batch_size=4, max_latency=0.5) # spawn child process as worker for multiple GPU # streamer = Streamer(predict_save, batch_size=4, max_latency=0.1) app.run(port=5005, debug=False, host='0.0.0.0') print('Flask started')