def get_tensor_from_files_list(files_list, device_id, size, pure_data_type):
ndatalist = []
for i, file_path in enumerate(files_list):
logger.debug("get tensor from filepath:{} i:{} of all:{}".format(
file_path, i, len(files_list)))
if file_path == pure_infer_dump_file:
ndata = get_pure_infer_data(size, pure_data_type)
elif file_path == None or os.path.exists(file_path) == False:
logger.error('filepath:{} not valid'.format(file_path))
raise RuntimeError()
#ndata = get_pure_infer_data(size)
elif file_path.endswith(".npy"):
ndata = np.load(file_path)
else:
with open(file_path, 'rb') as fd:
barray = fd.read()
ndata = np.frombuffer(barray, dtype=np.int8)
ndatalist.append(ndata)
ndata = np.concatenate(ndatalist)
if ndata.nbytes != size:
logger.error('ndata size:{} not match {}'.format(ndata.nbytes, size))
raise RuntimeError()
tensor = aclruntime.Tensor(ndata)
starttime = time.time()
tensor.to_device(device_id)
endtime = time.time()
summary.h2d_latency_list.append(float(endtime - starttime) *
1000.0) # millisecond
return tensor
def infer_step():
device_id = 0
options = aclruntime.session_options()
options.log_level = 1
session = aclruntime.InferenceSession(model_path, device_id, options)
# create new numpy data according inputs info
barray = bytearray(session.get_inputs()[0].realsize)
ndata = np.frombuffer(barray)
# convert numpy to pytensors in device
tensor = aclruntime.Tensor(ndata)
tensor.to_device(device_id)
outnames = [session.get_outputs()[0].name]
feeds = {session.get_inputs()[0].name: tensor}
# infer three step function
session.run_setinputs(feeds)
session.run_execute()
outputs = session.run_getoutputs(outnames)
# outputs = session.run(outnames, feeds)
print("step outputs:", outputs)
for out in outputs:
out.to_host()
# sumary inference throughput
print(session.sumary())
def infer_dynamicshape():
device_id = 0
options = aclruntime.session_options()
session = aclruntime.InferenceSession(model_path, device_id, options)
# only need call this functon compare infer_simple
session.set_dynamic_shape("actual_input_1:1,3,224,224")
session.set_custom_outsize([10000])
# create new numpy data according inputs info
barray = bytearray(session.get_inputs()[0].realsize)
ndata = np.frombuffer(barray)
# convert numpy to pytensors in device
tensor = aclruntime.Tensor(ndata)
tensor.to_device(device_id)
outnames = [session.get_outputs()[0].name]
feeds = {session.get_inputs()[0].name: tensor}
outputs = session.run(outnames, feeds)
print("outputs:", outputs)
for out in outputs:
out.to_host()
print(session.sumary())
def create_intensors_zerodata(intensors_desc, device_id):
intensors = []
for info in intensors_desc:
ndata = get_zero_ndata(info.realsize)
tensor = aclruntime.Tensor(ndata)
tensor.to_device(device_id)
intensors.append(tensor)
return intensors
def infer_run_simultaneous():
device_id = 0
# one call
options = aclruntime.session_options()
session = aclruntime.InferenceSession(model_path, device_id, options)
# create new numpy data according inputs info
barray = bytearray(session.get_inputs()[0].realsize)
ndata = np.frombuffer(barray)
# convert numpy to pytensors in device
tensor = aclruntime.Tensor(ndata)
tensor.to_device(device_id)
outnames = [session.get_outputs()[0].name]
feeds = {session.get_inputs()[0].name: tensor}
# another call
options1 = aclruntime.session_options()
model_path1 = sys.argv[2]
session1 = aclruntime.InferenceSession(model_path1, device_id, options1)
# create new numpy data according inputs info
barray1 = bytearray(session1.get_inputs()[0].realsize)
ndata1 = np.frombuffer(barray1)
# convert numpy to pytensors in device
tensor1 = aclruntime.Tensor(ndata1)
tensor1.to_device(device_id)
outnames1 = [session1.get_outputs()[0].name]
feeds1 = {session1.get_inputs()[0].name: tensor1}
# one run
outputs = session.run(outnames, feeds)
print("outputs:", outputs)
for out in outputs:
out.to_host()
# sumary inference throughput
print(session.sumary())
# another run
outputs1 = session1.run(outnames1, feeds1)
print("outputs1:", outputs1)
for out in outputs1:
out.to_host()
# sumary inference throughput
print(session1.sumary())
def test_infer_invalid_device_id(self):
device_id = 0
options = aclruntime.session_options()
model_path = self.get_resnet_static_om_path(1)
session = aclruntime.InferenceSession(model_path, device_id, options)
# create new numpy data according inputs info
barray = bytearray(session.get_inputs()[0].realsize)
ndata = np.frombuffer(barray)
# convert numpy to pytensors in device
tensor = aclruntime.Tensor(ndata)
with pytest.raises(RuntimeError) as e:
tensor.to_device(device_id + 100)
def create_intensors_zerodata(intensors_desc, device_id, pure_data_type):
intensors = []
for info in intensors_desc:
logger.debug("info shape:{} type:{} val:{} realsize:{} size:{}".format(
info.shape, info.datatype, int(info.datatype), info.realsize,
info.size))
ndata = get_pure_infer_data(info.realsize, pure_data_type)
tensor = aclruntime.Tensor(ndata)
starttime = time.time()
tensor.to_device(device_id)
endtime = time.time()
summary.h2d_latency_list.append(float(endtime - starttime) *
1000.0) # millisecond
intensors.append(tensor)
return intensors
def test_infer_invalid_outname(self):
device_id = 0
options = aclruntime.session_options()
model_path = self.get_resnet_static_om_path(1)
session = aclruntime.InferenceSession(model_path, device_id, options)
# create new numpy data according inputs info
barray = bytearray(session.get_inputs()[0].realsize)
ndata = np.frombuffer(barray)
# convert numpy to pytensors in device
tensor = aclruntime.Tensor(ndata)
tensor.to_device(device_id)
outnames = [session.get_outputs()[0].name + "xxx"]
feeds = {session.get_inputs()[0].name: tensor}
with pytest.raises(RuntimeError) as e:
outputs = session.run(outnames, feeds)
print("outputs:", outputs)
def test_infer_runcase_empty_outputname(self):
device_id = 0
options = aclruntime.session_options()
model_path = self.get_resnet_static_om_path(1)
session = aclruntime.InferenceSession(model_path, device_id, options)
# create new numpy data according inputs info
barray = bytearray(session.get_inputs()[0].realsize)
ndata = np.frombuffer(barray)
# convert numpy to pytensors in device
tensor = aclruntime.Tensor(ndata)
tensor.to_device(device_id)
outnames = []
feeds = [tensor]
outputs = session.run(outnames, feeds)
print("outputs:", outputs)
for out in outputs:
out.to_host()
print(session.sumary())
def test_infer_no_set_dynamicshape(self):
device_id = 0
options = aclruntime.session_options()
input_tensor_name = self.get_input_tensor_name()
model_path = self.get_resnet_dymshape_om_path()
session = aclruntime.InferenceSession(model_path, device_id, options)
session.set_custom_outsize([10000])
# create new numpy data according inputs info
barray = bytearray(session.get_inputs()[0].realsize)
ndata = np.frombuffer(barray)
# convert numpy to pytensors in device
tensor = aclruntime.Tensor(ndata)
tensor.to_device(device_id)
outnames = [ session.get_outputs()[0].name ]
feeds = { session.get_inputs()[0].name : tensor}
with pytest.raises(RuntimeError) as e:
outputs = session.run(outnames, feeds)
print("outputs:", outputs)
def predict(self, inputs):
intensors = []
for array in inputs:
cur_tensor = aclruntime.Tensor(array)
cur_tensor.to_device(self.device_id)
intensors.append(cur_tensor)
# outtensors = self.session.run(self.outputs, intensors)
self.session.run_setinputs(intensors)
start = ais_utils.get_datatime()
self.session.run_execute()
end = ais_utils.get_datatime()
outtensors = self.session.run_getoutputs(self.outputs)
self.elapsedtime += (end - start) * 1000
self.infercount += self.batchsize
outarrays = []
for tensor in outtensors:
tensor.to_host()
array = np.array(tensor)
outarrays.append(array)
return outarrays
