def get_tensor_from_files_list(files_list, device_id, size, pure_data_type): ndatalist = [] for i, file_path in enumerate(files_list): logger.debug("get tensor from filepath:{} i:{} of all:{}".format( file_path, i, len(files_list))) if file_path == pure_infer_dump_file: ndata = get_pure_infer_data(size, pure_data_type) elif file_path == None or os.path.exists(file_path) == False: logger.error('filepath:{} not valid'.format(file_path)) raise RuntimeError() #ndata = get_pure_infer_data(size) elif file_path.endswith(".npy"): ndata = np.load(file_path) else: with open(file_path, 'rb') as fd: barray = fd.read() ndata = np.frombuffer(barray, dtype=np.int8) ndatalist.append(ndata) ndata = np.concatenate(ndatalist) if ndata.nbytes != size: logger.error('ndata size:{} not match {}'.format(ndata.nbytes, size)) raise RuntimeError() tensor = aclruntime.Tensor(ndata) starttime = time.time() tensor.to_device(device_id) endtime = time.time() summary.h2d_latency_list.append(float(endtime - starttime) * 1000.0) # millisecond return tensor
def infer_step(): device_id = 0 options = aclruntime.session_options() options.log_level = 1 session = aclruntime.InferenceSession(model_path, device_id, options) # create new numpy data according inputs info barray = bytearray(session.get_inputs()[0].realsize) ndata = np.frombuffer(barray) # convert numpy to pytensors in device tensor = aclruntime.Tensor(ndata) tensor.to_device(device_id) outnames = [session.get_outputs()[0].name] feeds = {session.get_inputs()[0].name: tensor} # infer three step function session.run_setinputs(feeds) session.run_execute() outputs = session.run_getoutputs(outnames) # outputs = session.run(outnames, feeds) print("step outputs:", outputs) for out in outputs: out.to_host() # sumary inference throughput print(session.sumary())
def infer_dynamicshape(): device_id = 0 options = aclruntime.session_options() session = aclruntime.InferenceSession(model_path, device_id, options) # only need call this functon compare infer_simple session.set_dynamic_shape("actual_input_1:1,3,224,224") session.set_custom_outsize([10000]) # create new numpy data according inputs info barray = bytearray(session.get_inputs()[0].realsize) ndata = np.frombuffer(barray) # convert numpy to pytensors in device tensor = aclruntime.Tensor(ndata) tensor.to_device(device_id) outnames = [session.get_outputs()[0].name] feeds = {session.get_inputs()[0].name: tensor} outputs = session.run(outnames, feeds) print("outputs:", outputs) for out in outputs: out.to_host() print(session.sumary())
def create_intensors_zerodata(intensors_desc, device_id): intensors = [] for info in intensors_desc: ndata = get_zero_ndata(info.realsize) tensor = aclruntime.Tensor(ndata) tensor.to_device(device_id) intensors.append(tensor) return intensors
def infer_run_simultaneous(): device_id = 0 # one call options = aclruntime.session_options() session = aclruntime.InferenceSession(model_path, device_id, options) # create new numpy data according inputs info barray = bytearray(session.get_inputs()[0].realsize) ndata = np.frombuffer(barray) # convert numpy to pytensors in device tensor = aclruntime.Tensor(ndata) tensor.to_device(device_id) outnames = [session.get_outputs()[0].name] feeds = {session.get_inputs()[0].name: tensor} # another call options1 = aclruntime.session_options() model_path1 = sys.argv[2] session1 = aclruntime.InferenceSession(model_path1, device_id, options1) # create new numpy data according inputs info barray1 = bytearray(session1.get_inputs()[0].realsize) ndata1 = np.frombuffer(barray1) # convert numpy to pytensors in device tensor1 = aclruntime.Tensor(ndata1) tensor1.to_device(device_id) outnames1 = [session1.get_outputs()[0].name] feeds1 = {session1.get_inputs()[0].name: tensor1} # one run outputs = session.run(outnames, feeds) print("outputs:", outputs) for out in outputs: out.to_host() # sumary inference throughput print(session.sumary()) # another run outputs1 = session1.run(outnames1, feeds1) print("outputs1:", outputs1) for out in outputs1: out.to_host() # sumary inference throughput print(session1.sumary())
def test_infer_invalid_device_id(self): device_id = 0 options = aclruntime.session_options() model_path = self.get_resnet_static_om_path(1) session = aclruntime.InferenceSession(model_path, device_id, options) # create new numpy data according inputs info barray = bytearray(session.get_inputs()[0].realsize) ndata = np.frombuffer(barray) # convert numpy to pytensors in device tensor = aclruntime.Tensor(ndata) with pytest.raises(RuntimeError) as e: tensor.to_device(device_id + 100)
def create_intensors_zerodata(intensors_desc, device_id, pure_data_type): intensors = [] for info in intensors_desc: logger.debug("info shape:{} type:{} val:{} realsize:{} size:{}".format( info.shape, info.datatype, int(info.datatype), info.realsize, info.size)) ndata = get_pure_infer_data(info.realsize, pure_data_type) tensor = aclruntime.Tensor(ndata) starttime = time.time() tensor.to_device(device_id) endtime = time.time() summary.h2d_latency_list.append(float(endtime - starttime) * 1000.0) # millisecond intensors.append(tensor) return intensors
def test_infer_invalid_outname(self): device_id = 0 options = aclruntime.session_options() model_path = self.get_resnet_static_om_path(1) session = aclruntime.InferenceSession(model_path, device_id, options) # create new numpy data according inputs info barray = bytearray(session.get_inputs()[0].realsize) ndata = np.frombuffer(barray) # convert numpy to pytensors in device tensor = aclruntime.Tensor(ndata) tensor.to_device(device_id) outnames = [session.get_outputs()[0].name + "xxx"] feeds = {session.get_inputs()[0].name: tensor} with pytest.raises(RuntimeError) as e: outputs = session.run(outnames, feeds) print("outputs:", outputs)
def test_infer_runcase_empty_outputname(self): device_id = 0 options = aclruntime.session_options() model_path = self.get_resnet_static_om_path(1) session = aclruntime.InferenceSession(model_path, device_id, options) # create new numpy data according inputs info barray = bytearray(session.get_inputs()[0].realsize) ndata = np.frombuffer(barray) # convert numpy to pytensors in device tensor = aclruntime.Tensor(ndata) tensor.to_device(device_id) outnames = [] feeds = [tensor] outputs = session.run(outnames, feeds) print("outputs:", outputs) for out in outputs: out.to_host() print(session.sumary())
def test_infer_no_set_dynamicshape(self): device_id = 0 options = aclruntime.session_options() input_tensor_name = self.get_input_tensor_name() model_path = self.get_resnet_dymshape_om_path() session = aclruntime.InferenceSession(model_path, device_id, options) session.set_custom_outsize([10000]) # create new numpy data according inputs info barray = bytearray(session.get_inputs()[0].realsize) ndata = np.frombuffer(barray) # convert numpy to pytensors in device tensor = aclruntime.Tensor(ndata) tensor.to_device(device_id) outnames = [ session.get_outputs()[0].name ] feeds = { session.get_inputs()[0].name : tensor} with pytest.raises(RuntimeError) as e: outputs = session.run(outnames, feeds) print("outputs:", outputs)
def predict(self, inputs): intensors = [] for array in inputs: cur_tensor = aclruntime.Tensor(array) cur_tensor.to_device(self.device_id) intensors.append(cur_tensor) # outtensors = self.session.run(self.outputs, intensors) self.session.run_setinputs(intensors) start = ais_utils.get_datatime() self.session.run_execute() end = ais_utils.get_datatime() outtensors = self.session.run_getoutputs(self.outputs) self.elapsedtime += (end - start) * 1000 self.infercount += self.batchsize outarrays = [] for tensor in outtensors: tensor.to_host() array = np.array(tensor) outarrays.append(array) return outarrays