def test_onnx_conv_ns_ap_s2(self):
dcr = dc.conv(self.onnx_dc_X2, self.onnx_dc_W, self.dc_B, "NOTSET",
dc.vectorInt([]), 1, dc.vectorInt([]),
dc.vectorInt([1, 0, 1, 0]), dc.vectorInt([2, 2]))
np.testing.assert_allclose(self.onnx_npr_ap_s2.astype(np.float32),
np.array(dcr.data()).astype(np.float32),
rtol=1e-3,
atol=1e-3)
def test_onnx_conv_valid_s2(self):
dcr = dc.conv(self.onnx_dc_X2, self.onnx_dc_W, self.dc_B, "VALID",
dc.vectorInt([]), 1, dc.vectorInt([]), dc.vectorInt([]),
dc.vectorInt([2, 2]))
np.testing.assert_allclose(self.onnx_npr_vl_s2.astype(np.float32),
np.array(dcr.data()).astype(np.float32),
rtol=1e-3,
atol=1e-3)
def test_data(self):
# confirm type as class tuple.
a = dc.zeros(2, 3).asTypeInt()
adata = a.data()
assert type(adata) == type((1, ))
# load new data
new_data_list = [10, 11, 12, 13, 14, 15]
a.load(dc.vectorInt(new_data_list))
assert a[0] == 10
# load one element with flat index
a[0] = 777
assert a[0] == 777
# reshape, fetch and load with multi indices
a = dc.arange(12).asTypeInt()
a.reshape(dc.vectorSizeT([2, 2, 3]))
assert a[0, 1, 1] == 4
a[1, 1, 1] = 200
assert a[1, 1, 1] == 200
# negative test
try:
# This throws ValueError
print(a[0, 0, 9, 9, 9])
except ValueError as e:
assert e
def addParams(self, param):
if (param is None):
return None
if (len(param.FindInitializationErrors()) > 0):
print("WARNING (ONNX): initializer " + param.name +
" has following errors.\n")
print(" ", param.FindInitializationErrors())
print(" trying to load data with errors.\n")
param_type = dnnc.IR_DataType_NOTYPE
param_shape = dnnc.vectorSizeT(param.dims)
param_vec = None
param_vals = None
if param.data_type == param.INT8:
param_type = dnnc.IR_DataType_INT8
param_vals = [int(n) for n in param.int32_data]
if (len(param_vals) == 0):
param_vals = [int(n) for n in param.raw_data]
if (self._writeParamToDisk):
self.writeParamsToFile(param.name, param_vals)
param_vec = dnnc.vectorInt()
else:
param_vec = dnnc.vectorInt(param_vals)
elif param.data_type == param.INT16:
param_type = dnnc.IR_DataType_INT16
param_vals = [int(n) for n in param.int32_data]
if (len(param_vals) == 0):
param_vals = [int(n) for n in param.raw_data]
if (self._writeParamToDisk):
self.writeParamsToFile(param.name, param_vals)
param_vec = dnnc.vectorInt()
else:
param_vec = dnnc.vectorInt(param_vals)
elif param.data_type == param.INT32:
param_type = dnnc.IR_DataType_INT32
param_vals = [int(n) for n in param.int32_data]
if (len(param_vals) == 0):
param_vals = [int(n) for n in param.raw_data]
if (self._writeParamToDisk):
self.writeParamsToFile(param.name, param_vals)
param_vec = dnnc.vectorInt()
else:
param_vec = dnnc.vectorInt(param_vals)
elif param.data_type == param.INT64:
param_type = dnnc.IR_DataType_INT64
param_vals = [int(n) for n in param.int64_data]
if (len(param_vals) == 0):
param_vals = [int(n) for n in param.raw_data]
if (self._writeParamToDisk):
self.writeParamsToFile(param.name, param_vals)
param_vec = dnnc.vectorInt()
else:
param_vec = dnnc.vectorInt(param_vals)
elif param.data_type == param.UINT8:
param_type = dnnc.IR_DataType_UINT8
param_vals = [int(n) for n in param.uint64_data]
if (len(param_vals) == 0):
param_vals = [int(n) for n in param.raw_data]
if (self._writeParamToDisk):
self.writeParamsToFile(param.name, param_vals)
param_vec = dnnc.vectorInt()
else:
param_vec = dnnc.vectorInt(param_vals)
elif param.data_type == param.UINT16:
param_type = dnnc.IR_DataType_UINT16
param_vals = [int(n) for n in param.uint64_data]
if (len(param_vals) == 0):
param_vals = [int(n) for n in param.raw_data]
if (self._writeParamToDisk):
self.writeParamsToFile(param.name, param_vals)
param_vec = dnnc.vectorInt()
else:
param_vec = dnnc.vectorInt(param_vals)
elif param.data_type == param.UINT32:
param_type = dnnc.IR_DataType_UINT32
param_vals = [int(n) for n in param.uint64_data]
if (len(param_vals) == 0):
param_vals = [int(n) for n in param.raw_data]
if (self._writeParamToDisk):
self.writeParamsToFile(param.name, param_vals)
param_vec = dnnc.vectorInt()
else:
param_vec = dnnc.vectorInt(param_vals)
elif param.data_type == param.UINT64:
param_type = dnnc.IR_DataType_UINT64
param_vals = [int(n) for n in param.uint64_data]
if (len(param_vals) == 0):
param_vals = [int(n) for n in param.raw_data]
if (self._writeParamToDisk):
self.writeParamsToFile(param.name, param_vals)
param_vec = dnnc.vectorInt()
else:
param_vec = dnnc.vectorInt(param_vals)
elif param.data_type == param.FLOAT16:
param_type = dnnc.IR_DataType_FLOAT16
param_vals = [float(n) for n in param.float_data]
if (len(param_vals) == 0):
param_vals = [float(n) for n in param.raw_data]
if (self._writeParamToDisk):
self.writeParamsToFile(param.name, param_vals)
param_vec = dnnc.vectorInt()
else:
param_vec = dnnc.vectorFloat(param_vals)
elif param.data_type == param.BFLOAT16:
param_type = dnnc.IR_DataType_BFLOAT16
param_vals = [float(n) for n in param.float_data]
if (len(param_vals) == 0):
param_vals = [float(n) for n in param.raw_data]
if (self._writeParamToDisk):
self.writeParamsToFile(param.name, param_vals)
param_vec = dnnc.vectorFloat()
else:
param_vec = dnnc.vectorFloat(param_vals)
elif param.data_type == param.FLOAT:
param_type = dnnc.IR_DataType_FLOAT
param_vals = [float(n) for n in param.float_data]
if (len(param_vals) == 0):
param_vals = [float(n) for n in param.raw_data]
if (self._writeParamToDisk):
self.writeParamsToFile(param.name, param_vals)
param_vec = dnnc.vectorFloat()
else:
param_vec = dnnc.vectorFloat(param_vals)
elif param.data_type == param.DOUBLE:
param_type = dnnc.IR_DataType_DOUBLE
param_vals = [float(n) for n in param.double_data]
if (len(param_vals) == 0):
param_vals = [float(n) for n in param.raw_data]
if (self._writeParamToDisk):
self.writeParamsToFile(param.name, param_vals)
param_vec = dnnc.vectorFloat()
else:
param_vec = dnnc.vectorFloat(param_vals)
elif param.data_type == param.STRING:
param_type = dnnc.IR_DataType_STRING
param_vals = [str(s) for s in param.string_data]
if (len(param_vals) == 0):
param_vals = [str(n) for n in param.raw_data]
if (self._writeParamToDisk):
self.writeParamsToFile(param.name, param_vals)
param_vec = dnnc.vectorStr()
else:
param_vec = dnnc.vectorStr(param_vals)
elif param.data_type == param.BOOL:
param_type = dnnc.IR_DataType_BOOL
param_vals = [bool(b) for b in param.raw_data]
if (len(param_vals) == 0):
param_vals = [bool(b) for b in param.raw_data]
if (self._writeParamToDisk):
self.writeParamsToFile(param.name, param_vals)
param_vec = dnnc.vectorBool()
else:
param_vec = dnnc.vectorBool(param_vals)
else:
print("ERROR (ONNX): graph-node " + node.name + "\'s attribute " + \
param.name + " type " + str(param.data_type) + " is not valid.")
if (self._writeParamToDisk is False
and (param_type is dnnc.IR_DataType_NOTYPE or param_vec is None
or param_vec.size() == 0)):
print("ERROR (ONNX): did not find data for initializer ",
param.name)
return
param_irData = dnnc.irTypeData(param_type, param_vec)
dnnc_param = dnnc.dnnParameters(param.name, param_shape, param_irData)
self._dcGraph.addParameters(dnnc_param)
return dnnc_param
def addOPNode(self, node):
op_type = dnnc.getOpCode(node.op_type)
if (op_type is dnnc.opInvalid):
print("ERROR (ONNX):" + node.op_type +
" is not a valid graph-node op type.")
return None
dcNode = self._dcGraph.addOPNode(node.name, op_type)
for nd in node.input:
dcNode.addInput(nd)
for nd in node.output:
dcNode.addOutput(nd)
for attr in node.attribute:
attr_type = dnnc.IR_DataType_NOTYPE
attr_vals = []
attr_vec = None
if attr.type == onnx.AttributeProto.INT:
attr_type = dnnc.IR_DataType_INT32
attr_vals.append(attr.i)
attr_vec = dnnc.vectorInt(attr_vals)
elif attr.type == onnx.AttributeProto.INTS:
attr_type = dnnc.IR_DataType_INT32
for val in attr.ints:
attr_vals.append(int(val))
attr_vec = dnnc.vectorInt(attr_vals)
elif attr.type == onnx.AttributeProto.FLOAT:
attr_type = dnnc.IR_DataType_FLOAT
attr_vals.append(attr.f)
attr_vec = dnnc.vectorFloat(attr_vals)
elif attr.type == onnx.AttributeProto.FLOATS:
attr_type = dnnc.IR_DataType_FLOAT
for val in attr.floats:
attr_vals.append(float(val))
attr_vec = dnnc.vectorFloat(attr_vals)
elif attr.type == onnx.AttributeProto.STRING:
attr_type = dnnc.IR_DataType_STRING
attr_vals.append(str(attr.s))
attr_vec = dnnc.vectorStr(attr_vals)
elif attr.type == onnx.AttributeProto.STRINGS:
attr_type = dnnc.IR_DataType_STRING
for val in attr.strings:
attr_vals.append(str(val))
attr_vec = dnnc.vectorStr(attr_vals)
elif attr.type == onnx.AttributeProto.TENSOR:
if (attr.t.data_type == onnx.TensorProto.INT8
or attr.t.data_type == onnx.TensorProto.INT16
or attr.t.data_type == onnx.TensorProto.INT32
or attr.t.data_type == onnx.TensorProto.INT64):
attr_type = attr.t.data_type
attr_data = None
pack_format = 'P'
if (attr.t.data_type == onnx.TensorProto.INT8):
pack_format = 'b'
if (attr.t.data_type == onnx.TensorProto.INT16):
pack_format = 'h'
if (attr.t.data_type == onnx.TensorProto.INT32):
if (attr.t.int32_data):
attr_data = attr.t.int32_data
pack_format = 'i'
if (attr.t.data_type == onnx.TensorProto.INT64):
if (attr.t.int64_data):
attr_data = attr.t.int64_data
pack_format = 'q'
if (attr_data is None):
len = 1
for d in attr.t.dims:
len *= d
attr_data = struct.unpack(pack_format * len,
attr.t.raw_data)
if (attr_data is not None):
attr_tensor = dnnc.intTensor(attr.t.dims, attr.name)
attr_tensor.load(attr_data)
attr_vec = dnnc.vectorTensorInt()
attr_vec.push_back(attr_tensor)
else:
print("ERROR (ONNX): could not extract data for graph-node " + \
node.name + "\'s attribute " + attr.name + ".\n")
elif (attr.t.data_type == onnx.TensorProto.FLOAT16
or attr.t.data_type == onnx.TensorProto.FLOAT
or attr.t.data_type == onnx.TensorProto.DOUBLE):
attr_type = attr.t.data_type
attr_data = None
pack_format = 'P'
if (attr.t.data_type == onnx.TensorProto.FLOAT16):
if (attr.t.float_data):
attr_data = attr.t.float_data
pack_format = 'e'
if (attr.t.data_type == onnx.TensorProto.FLOAT):
if (attr.t.float_data):
attr_data = attr.t.float_data
pack_format = 'f'
if (attr.t.data_type == onnx.TensorProto.DOUBLE):
if (attr.t.double_data):
attr_data = attr.t.double_data
pack_format = 'd'
if (attr_data is None):
len = 1
for d in attr.t.dims:
len *= d
attr_data = struct.unpack(pack_format * len,
attr.t.raw_data)
if (attr_data is not None):
attr_tensor = dnnc.floatTensor(attr.t.dims, attr.name)
attr_tensor.load(attr_data)
attr_vec = dnnc.vectorTensorFloat()
attr_vec.push_back(attr_tensor)
else:
print("ERROR (ONNX): could not extract data for graph-node " + \
node.name + "\'s attribute " + attr.name + ".\n")
else:
print("ERROR (ONNX): attribute tensor's datatype " +
str(attr.t.data_type) + " isn't understood.")
elif attr.type == onnx.AttributeProto.TENSORS:
attr_type = dnnc.IR_DataType_TENSORS
attr_vals.append(attr.tensors)
attr_vec = dnnc.vectorTensorFloat(dnnc.floatTensor(attr_vals))
elif attr.type == onnx.AttributeProto.GRAPH:
attr_type = dnnc.IR_DataType_GRAPH
attr_vals.append(attr.g)
print(
"ERROR (ONNX): sub-graph in graph-node is not yet supported."
)
elif attr.type == onnx.AttributeProto.GRAPHS:
attr_type = dnnc.IR_DataType_GRAPH
attr_vals.append(attr.graphs)
print(
"ERROR (ONNX): sub-graph in graph-node is not yet supported."
)
else:
print("ERROR (ONNX): graph-node " + node.name + "\'s attribute " + \
attr.name + " type " + str(attr.type) + " is not valid.")
continue
if (attr_type is dnnc.IR_DataType_NOTYPE or attr_vec is None
or attr_vec.size() == 0):
print("ERROR (ONNX): graph-node " + node.name + "\'s attribute " + \
attr.name + " has no data.")
continue
attr_code = dnnc.getAttrName(attr.name)
if (attr_code is dnnc.attr_invalid):
print("WARN (ONNX): " + attr.name +
" is not a valid graph-node attribute.")
print(" operator " + node.op_type +
" will be added without this attribute.")
cAttrData = dnnc.irTypeData(attr_type, attr_vec)
cAttr = dnnc.nodeAttribute(attr_code, cAttrData)
dcNode.addAttribute(cAttr)
return dcNode
def addParams(self, param):
if ( param is None ):
return None;
if ( len(param.FindInitializationErrors()) > 0 ):
print("WARNING (ONNX): initializer " + param.name + " has following errors.\n");
print(" ", param.FindInitializationErrors());
print(" trying to load data with errors.\n");
param_type = dnnc.IR_DataType_NOTYPE;
param_shape = dnnc.vectorSizeT(param.dims)
param_vec = None
param_vals = None
param_len=1
for d in param.dims:
param_len *= d
if param.data_type == param.INT8 :
pack_format = 'b'
param_type = dnnc.IR_DataType_INT8;
param_vals = [int(n) for n in param.int32_data]
if ( len(param_vals) == 0 ):
param_vals = struct.unpack(pack_format*param_len, param.raw_data) ;
if ( self._writeParamToDisk ) :
self.writeParamsToFile(param.name, param_vals);
param_vec = dnnc.vectorInt()
else:
param_vec = dnnc.vectorInt(param_vals)
elif param.data_type == param.INT16 :
pack_format = 'h'
param_type = dnnc.IR_DataType_INT16;
param_vals = [int(n) for n in param.int32_data]
if ( len(param_vals) == 0 ):
param_vals = struct.unpack(pack_format*param_len, param.raw_data) ;
if ( self._writeParamToDisk ) :
self.writeParamsToFile(param.name, param_vals);
param_vec = dnnc.vectorInt()
else:
param_vec = dnnc.vectorInt(param_vals)
elif param.data_type == param.INT32:
pack_format = 'i'
param_type = dnnc.IR_DataType_INT32;
param_vals = [int(n) for n in param.int32_data]
if ( len(param_vals) == 0 ):
param_vals = struct.unpack(pack_format*param_len, param.raw_data) ;
if ( self._writeParamToDisk ) :
self.writeParamsToFile(param.name, param_vals);
param_vec = dnnc.vectorInt()
else:
param_vec = dnnc.vectorInt(param_vals)
elif param.data_type == param.INT64:
pack_format = 'q'
param_type = dnnc.IR_DataType_INT64;
param_vals = [int(n) for n in param.int64_data]
if ( len(param_vals) == 0 ):
param_vals = struct.unpack(pack_format*param_len, param.raw_data) ;
if ( self._writeParamToDisk ) :
self.writeParamsToFile(param.name, param_vals);
param_vec = dnnc.vectorInt()
else:
param_vec = dnnc.vectorInt(param_vals)
elif param.data_type == param.UINT8 :
pack_format = 'B'
param_type = dnnc.IR_DataType_UINT8;
param_vals = [int(n) for n in param.uint64_data]
if ( len(param_vals) == 0 ):
param_vals = struct.unpack(pack_format*param_len, param.raw_data) ;
if ( self._writeParamToDisk ) :
self.writeParamsToFile(param.name, param_vals);
param_vec = dnnc.vectorInt()
else:
param_vec = dnnc.vectorInt(param_vals)
elif param.data_type == param.UINT16 :
pack_format = 'H'
param_type = dnnc.IR_DataType_UINT16;
param_vals = [int(n) for n in param.uint64_data]
if ( len(param_vals) == 0 ):
param_vals = struct.unpack(pack_format*param_len, param.raw_data) ;
if ( self._writeParamToDisk ) :
self.writeParamsToFile(param.name, param_vals);
param_vec = dnnc.vectorInt()
else:
param_vec = dnnc.vectorInt(param_vals)
elif param.data_type == param.UINT32:
pack_format = 'I'
param_type = dnnc.IR_DataType_UINT32;
param_vals = [int(n) for n in param.uint64_data]
if ( len(param_vals) == 0 ):
param_vals = struct.unpack(pack_format*param_len, param.raw_data) ;
if ( self._writeParamToDisk ) :
self.writeParamsToFile(param.name, param_vals);
param_vec = dnnc.vectorInt()
else:
param_vec = dnnc.vectorInt(param_vals)
elif param.data_type == param.UINT64:
pack_format = 'L'
param_type = dnnc.IR_DataType_UINT64;
param_vals = [int(n) for n in param.uint64_data]
if ( len(param_vals) == 0 ):
param_vals = struct.unpack(pack_format*param_len, param.raw_data) ;
if ( self._writeParamToDisk ) :
self.writeParamsToFile(param.name, param_vals);
param_vec = dnnc.vectorInt()
else:
param_vec = dnnc.vectorInt(param_vals)
elif param.data_type == param.FLOAT16 :
pack_format = 'e'
param_type = dnnc.IR_DataType_FLOAT16;
param_vals = [float(n) for n in param.float_data]
if ( len(param_vals) == 0 ):
param_vals = struct.unpack(pack_format*param_len, param.raw_data) ;
if ( self._writeParamToDisk ) :
self.writeParamsToFile(param.name, param_vals);
param_vec = dnnc.vectorInt()
else:
param_vec = dnnc.vectorFloat(param_vals)
elif param.data_type == param.BFLOAT16 :
pack_format = 'e'
param_type = dnnc.IR_DataType_BFLOAT16;
param_vals = [float(n) for n in param.float_data]
if ( len(param_vals) == 0 ):
param_vals = struct.unpack(pack_format*param_len, param.raw_data) ;
if ( self._writeParamToDisk ) :
self.writeParamsToFile(param.name, param_vals);
param_vec = dnnc.vectorFloat()
else:
param_vec = dnnc.vectorFloat(param_vals)
elif param.data_type == param.FLOAT:
pack_format = 'f'
param_type = dnnc.IR_DataType_FLOAT;
param_vals = [float(n) for n in param.float_data]
if ( len(param_vals) == 0 ):
param_vals = struct.unpack(pack_format*param_len, param.raw_data) ;
if ( self._writeParamToDisk ) :
self.writeParamsToFile(param.name, param_vals);
param_vec = dnnc.vectorFloat()
else:
param_vec = dnnc.vectorFloat(param_vals)
elif param.data_type == param.DOUBLE:
pack_format = 'd'
param_type = dnnc.IR_DataType_DOUBLE;
param_vals = [float(n) for n in param.double_data]
if ( len(param_vals) == 0 ):
param_vals = struct.unpack(pack_format*param_len, param.raw_data) ;
if ( self._writeParamToDisk ) :
self.writeParamsToFile(param.name, param_vals);
param_vec = dnnc.vectorFloat()
else:
param_vec = dnnc.vectorFloat(param_vals)
elif param.data_type == param.STRING:
pack_format = 's'
param_type = dnnc.IR_DataType_STRING;
param_vals = [str(s) for s in param.string_data]
if ( len(param_vals) == 0 ):
param_vals = struct.unpack(pack_format*param_len, param.raw_data) ;
if ( self._writeParamToDisk ) :
self.writeParamsToFile(param.name, param_vals);
param_vec = dnnc.vectorStr()
else:
param_vec = dnnc.vectorStr(param_vals)
elif param.data_type == param.BOOL:
pack_format = '?'
param_type = dnnc.IR_DataType_BOOL;
param_vals = [bool(b) for b in param.raw_data]
if ( len(param_vals) == 0 ):
param_vals = struct.unpack(pack_format*param_len, param.raw_data) ;
if ( self._writeParamToDisk ) :
self.writeParamsToFile(param.name, param_vals);
param_vec = dnnc.vectorBool()
else:
param_vec = dnnc.vectorBool(param_vals)
else:
print("ERROR (ONNX): graph-node " + node.name + "\'s attribute " + \
param.name + " type " + str(param.data_type) + " is not valid.")
if ( self._writeParamToDisk is False and
(param_type is dnnc.IR_DataType_NOTYPE or param_vec is None or param_vec.size()==0) ) :
print("ERROR (ONNX): did not find data for initializer ", param.name);
return;
param_irData = dnnc.irTypeData(param_type, param_vec) ;
dnnc_param = dnnc.dnnParameters(param.name, param_shape, param_irData);
self._dcGraph.addParameters(dnnc_param) ;
return dnnc_param;