def _main(): context = PredictContext(FLAGS.model_name, url=FLAGS.url, protocol=FLAGS.protocol, verbose=True) model_config = context.model_config input_name, output_name, c, h, w, data_format, dtype = _parse_model( model_config, FLAGS.model_name, FLAGS.batch_size) file_names, image_data = _gen_input_data(data_format, dtype, c, h, w) cur_idx = 0 num_of_images = len(image_data) def _next_batch(batch_size): nonlocal cur_idx if cur_idx + batch_size <= num_of_images: inputs = image_data[cur_idx:cur_idx + batch_size] outputs = file_names[cur_idx:cur_idx + batch_size] cur_idx = (cur_idx + batch_size) % num_of_images else: image_idx = cur_idx cur_idx = 0 next_inputs, next_outputs = _next_batch(batch_size - (num_of_images - image_idx)) inputs = image_data[image_idx:] + next_inputs outputs = file_names[image_idx:] + next_outputs return inputs, outputs num_of_batches = 99 for _ in range(num_of_batches): i_inputs, i_outputs = _next_batch(FLAGS.batch_size) context.run(inputs={input_name: i_inputs}, outputs={output_name: FLAGS.classes}, batch_size=FLAGS.batch_size)
def _main(): context = PredictContext(FLAGS.model_name, url=FLAGS.url, protocol=FLAGS.protocol, verbose=True) image_data = [] image = np.load('./input_0.npy') image_data.append(image) input_name = context.model_config.input[0].name output_name = 'save_infer_model/scale_0.tmp_1' cur_idx = 0 num_of_images = len(image_data) def _next_batch(batch_size): nonlocal cur_idx if cur_idx + batch_size <= num_of_images: inputs = image_data[cur_idx:cur_idx + batch_size] cur_idx = (cur_idx + batch_size) % num_of_images else: image_idx = cur_idx cur_idx = 0 next_inputs = _next_batch(batch_size - (num_of_images - image_idx)) inputs = image_data[image_idx:] + next_inputs return inputs num_of_batches = 100 for _ in range(num_of_batches): i_inputs = _next_batch(FLAGS.batch_size) context.run(inputs={input_name: i_inputs}, outputs=[output_name], batch_size=FLAGS.batch_size)
def _main(): context = PredictContext(FLAGS.model_name, url=FLAGS.url, protocol=FLAGS.protocol, verbose=True) model_config = context.model_config input_name, output_name, c, h, w, data_format, dtype = _parse_model( model_config, FLAGS.model_name, FLAGS.batch_size) file_names, image_data = _gen_input_data(data_format, dtype, c, h, w) cur_idx = 0 num_of_images = len(image_data) def _next_batch(batch_size): nonlocal cur_idx if cur_idx + batch_size <= num_of_images: inputs = image_data[cur_idx:cur_idx + batch_size] outputs = file_names[cur_idx:cur_idx + batch_size] cur_idx = (cur_idx + batch_size) % num_of_images else: image_idx = cur_idx cur_idx = 0 next_inputs, next_outputs = _next_batch(batch_size - (num_of_images - image_idx)) inputs = image_data[image_idx:] + next_inputs outputs = file_names[image_idx:] + next_outputs return inputs, outputs num_of_batches = 99 if num_of_images % FLAGS.batch_size != 0: num_of_batches += 1 logging.basicConfig( level=logging.DEBUG, filename=os.path.join(FLAGS.log_path, 'client_time.log'), filemode='a', format= '%(asctime)s - %(pathname)s[line:%(lineno)d] - %(levelname)s: %(message)s' ) for _ in range(num_of_batches): i_inputs, i_outputs = _next_batch(FLAGS.batch_size) time1 = time.time() context.run(inputs={input_name: i_inputs}, outputs={output_name: FLAGS.classes}, batch_size=FLAGS.batch_size) logging.info(f'The time of predict: {time.time() - time1}') print(f'{_} / {num_of_batches}')
def _test_connect_of_serving(args): try: PredictContext(os.getenv('MODEL_NAME'), url=args.url, protocol=args.protocol, verbose=True) return True except _InactiveRpcError: return False
def _main(): context = PredictContext(FLAGS.model_name, url=FLAGS.url, protocol=FLAGS.protocol, verbose=True) model_config = context.model_config input_name, output_name = _parse_model(model_config, FLAGS.model_name, FLAGS.batch_size) file_names, image_data = _gen_input_data() cur_idx = 0 num_of_images = len(image_data) labels = open(FLAGS.label_file).read().strip().split('\n') def _next_batch(batch_size): nonlocal cur_idx if cur_idx + batch_size <= num_of_images: inputs = image_data[cur_idx:cur_idx + batch_size] outputs = file_names[cur_idx:cur_idx + batch_size] cur_idx = (cur_idx + batch_size) % num_of_images else: image_idx = cur_idx cur_idx = 0 next_inputs, next_outputs = _next_batch(batch_size - (num_of_images - image_idx)) inputs = image_data[image_idx:] + next_inputs outputs = file_names[image_idx:] + next_outputs return inputs, outputs num_of_batches = num_of_images // FLAGS.batch_size if num_of_images % FLAGS.batch_size != 0: num_of_batches += 1 for _ in range(num_of_batches): i_inputs, i_outputs = _next_batch(FLAGS.batch_size) result = context.run(inputs={input_name: i_inputs}, outputs={output_name: FLAGS.classes}, batch_size=FLAGS.batch_size) _postprocess(result[output_name], i_outputs, FLAGS.batch_size, labels)
def _main(): context = PredictContext(FLAGS.model_name, url=FLAGS.url, protocol=FLAGS.protocol, verbose=True) image_data = np.load('./ppyolo_inputs/input_0.npy') input_size = np.load('./ppyolo_inputs/input_1.npy') input_name1 = context.model_config.input[0].name input_name2 = context.model_config.input[1].name output_name1 = context.model_config.output[0].name output_name2 = context.model_config.output[1].name cur_idx = 0 num_of_images = len(image_data) def _next_batch(batch_size): nonlocal cur_idx if cur_idx + batch_size <= num_of_images: inputs_1 = image_data[cur_idx:cur_idx + batch_size] inputs_2 = input_size[cur_idx:cur_idx + batch_size] cur_idx = (cur_idx + batch_size) % num_of_images else: image_idx = cur_idx cur_idx = 0 next_inputs_1, next_inputs_2 = _next_batch(batch_size - (num_of_images - image_idx)) inputs_1 = image_data[image_idx:] + next_inputs_1 inputs_2 = input_size[image_idx:] + next_inputs_2 return inputs_1, inputs_2 num_of_batches = 1 if num_of_images % FLAGS.batch_size != 0: num_of_batches += 1 for _ in range(num_of_batches): i_inputs_1, i_inputs_2 = _next_batch(FLAGS.batch_size) context.run(inputs={input_name1: i_inputs_1, input_name2: i_inputs_2}, outputs=[output_name1, output_name2], batch_size=FLAGS.batch_size)
FLAGS = parser.parse_args() protocol = FLAGS.protocol # We use a simple model that takes 2 input tensors of 16 integers # each and returns 2 output tensors of 16 integers each. One # output tensor is the element-wise sum of the inputs and one # output is the element-wise difference. model_name = "plan_float16_float32_float32" model_version = 1 batch_size = 1 input0_data = np.arange(start=0, stop=16, dtype=np.float16) input1_data = np.ones(shape=16, dtype=np.float16) context = PredictContext(model_name, model_version=model_version, url=FLAGS.url, protocol=FLAGS.protocol) result = context.run(inputs={ 'INPUT0': (input0_data, ), 'INPUT1': (input1_data, ) }, outputs=['OUTPUT0', 'OUTPUT1'], batch_size=batch_size) output0_data = result['OUTPUT0'].tensor[0] output1_data = result['OUTPUT1'].tensor[0] for i in range(16): print( str(input0_data[i]) + " + " + str(input1_data[i]) + " = " + str(output0_data[i]))
def _main(): credentials = None if FLAGS.protocol == 'grpc' and FLAGS.credentials is not None: with open(FLAGS.credentials, 'rb') as f: credentials = f.read() context = PredictContext(FLAGS.model_name, model_version=FLAGS.model_version, signature=FLAGS.signature, url=FLAGS.url, protocol=FLAGS.protocol, grpc_domain=FLAGS.grpc_domain, credentials=credentials, verbose=FLAGS.verbose) input_name, output_name, c, h, w, data_format, dtype, output_size = parse_model( context.model_config, FLAGS.model_name, FLAGS.batch_size) print('Input name: {}, output name: {}, (c, h, w)=({}, {}, {}), data_format: {}, dtype: {}, output size: {}'.format( input_name, output_name, c, h, w, data_format, dtype, output_size)) if os.path.isdir(FLAGS.image_filename): filenames = [os.path.join(FLAGS.image_filename, f) for f in os.listdir(FLAGS.image_filename) if os.path.isfile(os.path.join(FLAGS.image_filename, f))] else: filenames = [FLAGS.image_filename, ] filenames.sort() image_data = [] for filename in filenames: img = Image.open(filename) array = preprocess(img, data_format, dtype, c, h, w, FLAGS.scaling) image_data.append(array) cur_idx = 0 num_of_images = len(image_data) def _next_batch(batch_size): nonlocal cur_idx if cur_idx + batch_size <= num_of_images: inputs = image_data[cur_idx:cur_idx + batch_size] outputs = filenames[cur_idx:cur_idx + batch_size] cur_idx = (cur_idx + batch_size) % num_of_images else: image_idx = cur_idx cur_idx = 0 next_inputs, next_outputs = _next_batch(batch_size - (num_of_images - image_idx)) inputs = image_data[image_idx:] + next_inputs outputs = filenames[image_idx:] + next_outputs return inputs, outputs num_of_batches = num_of_images // FLAGS.batch_size if num_of_images % FLAGS.batch_size != 0: num_of_batches += 1 if FLAGS.protocol == 'grpc' and FLAGS.is_async: print("async inference ") requests = [] batchOutputs = [] # async submit input data use the grpc inferface for _ in range(num_of_batches): i_inputs, i_outputs = _next_batch(FLAGS.batch_size) batchOutputs.append(i_outputs) requests.append(context.async_run(inputs={input_name: i_inputs}, outputs={output_name: FLAGS.classes}, batch_size=FLAGS.batch_size)) for i in range(len(requests)): response = requests[i].result() result = context.process_response({output_name: FLAGS.classes}, response) postprocess(result[output_name], batchOutputs[i], FLAGS.batch_size) else: for _ in range(num_of_batches): i_inputs, i_outputs = _next_batch(FLAGS.batch_size) result = context.run(inputs={input_name: i_inputs}, outputs={output_name: FLAGS.classes}, batch_size=FLAGS.batch_size) postprocess(result[output_name], i_outputs, FLAGS.batch_size)