def test_model23(self):
try:
import onnx
except:
unittest.TestCase.skipTest(self, "onnx not found in the libraries")
model1 = Sequential(self.s, model_table='Simple_CNN1')
model1.add(InputLayer(3, 224, 224))
model1.add(Conv2d(8, 7, act='identity', include_bias=False))
model1.add(BN(act='relu'))
model1.add(Pooling(2))
model1.add(Conv2d(8, 7, act='identity', include_bias=False))
model1.add(BN(act='relu'))
model1.add(Pooling(2))
model1.add(Dense(2))
model1.add(OutputLayer(act='softmax', n=2))
if self.data_dir is None:
unittest.TestCase.skipTest(self, "DLPY_DATA_DIR is not set in the environment variables")
caslib, path = caslibify(self.s, path=self.data_dir+'images.sashdat', task='load')
self.s.table.loadtable(caslib=caslib,
casout={'name': 'eee', 'replace': True},
path=path)
r = model1.fit(data='eee', inputs='_image_', target='_label_', max_epochs=1)
self.assertTrue(r.severity == 0)
model1.deploy(self.data_dir, output_format='onnx')
def test_model22_1(self):
try:
import onnx
except:
unittest.TestCase.skipTest(self, "onnx not found in the libraries")
from onnx import numpy_helper
import numpy as np
model1 = Sequential(self.s, model_table='Simple_CNN1')
model1.add(InputLayer(3, 224, 224))
model1.add(Conv2d(8, 7, act='identity', include_bias=False))
model1.add(Reshape(height=448, width=448, depth=2))
model1.add(Dense(2))
model1.add(OutputLayer(act='softmax', n=2))
if self.data_dir is None:
unittest.TestCase.skipTest(self, "DLPY_DATA_DIR is not set in the environment variables")
caslib, path = caslibify(self.s, path=self.data_dir+'images.sashdat', task='load')
self.s.table.loadtable(caslib=caslib,
casout={'name': 'eee', 'replace': True},
path=path)
r = model1.fit(data='eee', inputs='_image_', target='_label_', max_epochs=1)
self.assertTrue(r.severity == 0)
model1.deploy(self.data_dir_local, output_format='onnx')
model_path = os.path.join(self.data_dir_local, 'Simple_CNN1.onnx')
m = onnx.load(model_path)
self.assertEqual(m.graph.node[1].op_type, 'Reshape')
init = numpy_helper.to_array(m.graph.initializer[1])
self.assertTrue(np.array_equal(init, [ -1, 2, 448, 448]))
def test_model18(self):
model1 = Sequential(self.s, model_table='Simple_CNN1')
model1.add(InputLayer(3, 224, 224))
model1.add(Conv2d(8, 7))
model1.add(Pooling(2))
model1.add(Conv2d(8, 7))
model1.add(Pooling(2))
model1.add(Dense(16))
model1.add(OutputLayer(act='softmax', n=2))
if self.data_dir is None:
unittest.TestCase.skipTest(
self, "DLPY_DATA_DIR is not set in the environment variables")
caslib, path = caslibify(self.s,
path=self.data_dir + 'images.sashdat',
task='load')
self.s.table.loadtable(caslib=caslib,
casout={
'name': 'eee',
'replace': True
},
path=path)
r = model1.fit(data='eee',
inputs='_image_',
target='_label_',
max_epochs=1)
self.assertTrue(r.severity == 0)
model1.save_weights_csv(self.data_dir)
def test_astore_deploy_wrong_path(self):
from dlpy.sequential import Sequential
from dlpy.utils import caslibify
model1 = Sequential(self.s, model_table='Simple_CNN1')
model1.add(InputLayer(3, 224, 224))
model1.add(Conv2d(1, 17))
model1.add(Pooling(14))
model1.add(Dense(3))
model1.add(OutputLayer(act='softmax', n=2))
if self.data_dir is None:
tm.TestCase.skipTest(
self, "DLPY_DATA_DIR is not set in the environment variables")
caslib, path, tmp_caslib = caslibify(self.s,
path=self.data_dir +
'images.sashdat',
task='load')
self.s.table.loadtable(caslib=caslib,
casout={
'name': 'eee',
'replace': True
},
path=path)
r = model1.fit(data='eee',
inputs='_image_',
target='_label_',
save_best_weights=True)
self.assertTrue(r.severity == 0)
with self.assertRaises(DLPyError):
model1.deploy(path='/amran/komran', output_format='astore')
def test_model12(self):
model1 = Sequential(self.s, model_table='Simple_CNN1')
model1.add(InputLayer(3, 224, 224))
model1.add(Conv2d(8, 7))
model1.add(Pooling(2))
model1.add(Conv2d(8, 7))
model1.add(Pooling(2))
model1.add(Dense(16))
model1.add(OutputLayer(act='softmax', n=2))
if self.data_dir is None:
unittest.TestCase.skipTest(self, "DLPY_DATA_DIR is not set in the environment variables")
caslib, path, tmp_caslib = caslibify(self.s, path=self.data_dir+'images.sashdat', task='load')
self.s.table.loadtable(caslib=caslib,
casout={'name': 'eee', 'replace': True},
path=path)
r = model1.fit(data='eee', inputs='_image_', target='_label_', save_best_weights=True)
self.assertTrue(r.severity == 0)
r1 = model1.fit(data='eee', inputs='_image_', target='_label_', max_epochs=3)
self.assertTrue(r1.severity == 0)
r2 = model1.fit(data='eee', inputs='_image_', target='_label_', max_epochs=2, save_best_weights=True)
self.assertTrue(r2.severity == 0)
r3 = model1.predict(data='eee', use_best_weights=True)
self.assertTrue(r3.severity == 0)
if (caslib is not None) and tmp_caslib:
self.s.retrieve('table.dropcaslib', message_level = 'error', caslib = caslib)
def test_model1(self):
model1 = Sequential(self.s, model_table='Simple_CNN1')
model1.add(InputLayer(3, 224, 224))
model1.add(Conv2d(8, 7))
model1.add(Pooling(2))
model1.add(Conv2d(8, 7))
model1.add(Pooling(2))
model1.add(Dense(16))
model1.add(OutputLayer(act='softmax', n=2))
if self.data_dir is None:
unittest.TestCase.skipTest(self, "DLPY_DATA_DIR is not set in the environment variables")
caslib, path, tmp_caslib = caslibify(self.s, path=self.data_dir+'images.sashdat', task='load')
self.s.table.loadtable(caslib=caslib,
casout={'name': 'eee', 'replace': True},
path=path)
r = model1.fit(data='eee', inputs='_image_', target='_label_', lr=0.001)
if r.severity > 0:
for msg in r.messages:
print(msg)
self.assertTrue(r.severity <= 1)
if (caslib is not None) and tmp_caslib:
self.s.retrieve('table.dropcaslib', message_level = 'error', caslib = caslib)
def test_CyclicLR(self):
model1 = Sequential(self.s, model_table = 'Simple_CNN1')
model1.add(InputLayer(3, 224, 224))
model1.add(Conv2d(8, 7))
model1.add(Pooling(2))
model1.add(Conv2d(8, 7))
model1.add(Pooling(2))
model1.add(Dense(16))
model1.add(OutputLayer(act = 'softmax', n = 2))
if self.data_dir is None:
unittest.TestCase.skipTest(self, "DLPY_DATA_DIR is not set in the environment variables")
caslib, path, tmp_caslib = caslibify(self.s, path = self.data_dir + 'images.sashdat', task = 'load')
self.s.table.loadtable(caslib = caslib,
casout = {'name': 'eee', 'replace': True},
path = path)
lrs = CyclicLR(self.s, 'eee', 4, 1.0, 0.0000001, 0.01)
solver = VanillaSolver(lr_scheduler=lrs)
self.assertTrue(self.sample_syntax['CyclicLR'] == solver)
optimizer = Optimizer(algorithm = solver, log_level = 3, max_epochs = 4, mini_batch_size = 2)
r = model1.fit(data = 'eee', inputs = '_image_', target = '_label_', optimizer = optimizer, n_threads=2)
if r.severity > 0:
for msg in r.messages:
print(msg)
self.assertTrue(r.severity <= 1)
def test_model22(self):
model1 = Sequential(self.s, model_table='Simple_CNN1')
model1.add(InputLayer(3, 224, 224))
model1.add(Conv2d(8, 7))
pool1 = Pooling(2)
model1.add(pool1)
conv1 = Conv2d(1, 1, act='identity', src_layers=[pool1])
model1.add(conv1)
model1.add(Res(act='relu', src_layers=[conv1, pool1]))
model1.add(Pooling(2))
model1.add(Dense(2))
model1.add(OutputLayer(act='softmax', n=2))
if self.data_dir is None:
unittest.TestCase.skipTest(self, "DLPY_DATA_DIR is not set in the environment variables")
caslib, path = caslibify(self.s, path=self.data_dir+'images.sashdat', task='load')
self.s.table.loadtable(caslib=caslib,
casout={'name': 'eee', 'replace': True},
path=path)
r = model1.fit(data='eee', inputs='_image_', target='_label_', max_epochs=1)
self.assertTrue(r.severity == 0)
model1.deploy(self.data_dir, output_format='onnx')
def test_model15(self):
# test RECTIFIER activation for concat layer
try:
import onnx
except:
unittest.TestCase.skipTest(self, "onnx not found in the libraries")
model1 = Sequential(self.s, model_table='Simple_CNN1')
model1.add(InputLayer(3, 224, 224))
model1.add(Conv2d(8, 7))
pool1 = Pooling(2)
model1.add(pool1)
conv1 = Conv2d(1, 7, src_layers=[pool1])
conv2 = Conv2d(1, 7, src_layers=[pool1])
model1.add(conv1)
model1.add(conv2)
model1.add(Concat(act='RECTIFIER', src_layers=[conv1, conv2]))
model1.add(Pooling(2))
model1.add(Dense(2))
model1.add(OutputLayer(act='softmax', n=2))
if self.data_dir is None:
unittest.TestCase.skipTest(
self, "DLPY_DATA_DIR is not set in the environment variables")
caslib, path, tmp_caslib = caslibify(self.s,
path=self.data_dir +
'images.sashdat',
task='load')
self.s.table.loadtable(caslib=caslib,
casout={
'name': 'eee',
'replace': True
},
path=path)
r = model1.fit(data='eee',
inputs='_image_',
target='_label_',
max_epochs=1)
self.assertTrue(r.severity == 0)
import tempfile
tmp_dir_to_dump = tempfile.gettempdir()
model1.deploy(tmp_dir_to_dump, output_format='onnx')
import os
os.remove(os.path.join(tmp_dir_to_dump, "Simple_CNN1.onnx"))
if (caslib is not None) and tmp_caslib:
self.s.retrieve('table.dropcaslib',
message_level='error',
caslib=caslib)
def test_model13(self):
# test dropout
try:
import onnx
except:
unittest.TestCase.skipTest(self, "onnx not found in the libraries")
model1 = Sequential(self.s, model_table='Simple_CNN1')
model1.add(InputLayer(3, 224, 224))
model1.add(
Conv2d(8, 7, act='IDENTITY', dropout=0.5, include_bias=False))
model1.add(BN(act='relu'))
model1.add(Pooling(2, pool='MEAN', dropout=0.5))
model1.add(
Conv2d(8, 7, act='IDENTITY', dropout=0.5, include_bias=False))
model1.add(BN(act='relu'))
model1.add(Pooling(2, pool='MEAN', dropout=0.5))
model1.add(Conv2d(8, 7, act='identity', include_bias=False))
model1.add(BN(act='relu'))
model1.add(Dense(16, act='IDENTITY', dropout=0.1))
model1.add(OutputLayer(act='softmax', n=2))
if self.data_dir is None:
unittest.TestCase.skipTest(
self, "DLPY_DATA_DIR is not set in the environment variables")
caslib, path, tmp_caslib = caslibify(self.s,
path=self.data_dir +
'images.sashdat',
task='load')
self.s.table.loadtable(caslib=caslib,
casout={
'name': 'eee',
'replace': True
},
path=path)
r = model1.fit(data='eee',
inputs='_image_',
target='_label_',
max_epochs=2)
self.assertTrue(r.severity == 0)
import tempfile
tmp_dir_to_dump = tempfile.gettempdir()
model1.deploy(tmp_dir_to_dump, output_format='onnx')
import os
os.remove(os.path.join(tmp_dir_to_dump, "Simple_CNN1.onnx"))
if (caslib is not None) and tmp_caslib:
self.s.retrieve('table.dropcaslib',
message_level='error',
caslib=caslib)
def test_imagescaler2(self):
# test export model with imagescaler
try:
import onnx
except:
unittest.TestCase.skipTest(self, 'onnx not found')
if self.data_dir_local is None:
unittest.TestCase.skipTest(
self, 'DLPY_DATA_DIR_LOCAL is not set in '
'the environment variables')
model1 = Sequential(self.s, model_table='imagescaler2')
model1.add(
InputLayer(n_channels=3,
width=224,
height=224,
scale=1 / 255.,
offsets=[0.1, 0.2, 0.3]))
model1.add(Conv2d(8, 7))
model1.add(Pooling(2))
model1.add(OutputLayer(act='softmax', n=2))
caslib, path = caslibify(self.s,
path=self.data_dir + 'images.sashdat',
task='load')
self.s.table.loadtable(caslib=caslib,
casout={
'name': 'eee',
'replace': True
},
path=path)
r = model1.fit(data='eee',
inputs='_image_',
target='_label_',
max_epochs=1)
self.assertTrue(r.severity == 0)
from dlpy.model_conversion.write_onnx_model import sas_to_onnx
onnx_model = sas_to_onnx(model1.layers,
self.s.CASTable('imagescaler2'),
self.s.CASTable('imagescaler2_weights'))
self.assertAlmostEqual(onnx_model.graph.node[0].attribute[0].floats[0],
0.1)
self.assertAlmostEqual(onnx_model.graph.node[0].attribute[0].floats[1],
0.2)
self.assertAlmostEqual(onnx_model.graph.node[0].attribute[0].floats[2],
0.3)
self.assertAlmostEqual(onnx_model.graph.node[0].attribute[1].f,
1 / 255.)
def test_evaluate_obj_det(self):
if self.data_dir is None:
unittest.TestCase.skipTest(
self, "DLPY_DATA_DIR is not set in the environment variables")
caslib, path = caslibify(self.s,
path=self.data_dir +
'evaluate_obj_det_det.sashdat',
task='load')
self.s.table.loadtable(caslib=caslib,
casout={
'name': 'evaluate_obj_det_det',
'replace': True
},
path=path)
self.s.table.loadtable(caslib=caslib,
casout={
'name': 'evaluate_obj_det_gt',
'replace': True
},
path='evaluate_obj_det_gt.sashdat')
yolo_anchors = (5.9838598901098905, 3.4326923076923075,
2.184993862520458, 1.9841448445171848,
1.0261752136752136, 1.2277777777777779)
yolo_model = Tiny_YoloV2(self.s,
grid_number=17,
scale=1.0 / 255,
n_classes=1,
height=544,
width=544,
predictions_per_grid=3,
anchors=yolo_anchors,
max_boxes=100,
coord_type='yolo',
max_label_per_image=100,
class_scale=1.0,
coord_scale=2.0,
prediction_not_a_object_scale=1,
object_scale=5,
detection_threshold=0.05,
iou_threshold=0.2)
metrics = yolo_model.evaluate_object_detection(
ground_truth='evaluate_obj_det_gt',
coord_type='yolo',
detection_data='evaluate_obj_det_det',
iou_thresholds=0.5)
def test_plot_ticks(self):
model1 = Sequential(self.s, model_table='Simple_CNN1')
model1.add(InputLayer(3, 224, 224))
model1.add(Conv2d(8, 7))
model1.add(Pooling(2))
model1.add(Conv2d(8, 7))
model1.add(Pooling(2))
model1.add(Dense(16))
model1.add(OutputLayer(act='softmax', n=2))
if self.data_dir is None:
unittest.TestCase.skipTest(self, "DLPY_DATA_DIR is not set in the environment variables")
caslib, path, tmp_caslib = caslibify(self.s, path=self.data_dir+'images.sashdat', task='load')
self.s.table.loadtable(caslib=caslib,
casout={'name': 'eee', 'replace': True},
path=path)
r = model1.fit(data='eee', inputs='_image_', target='_label_', lr=0.001, max_epochs=5)
# Test default tick_frequency value of 1
ax = model1.plot_training_history()
self.assertEqual(len(ax.xaxis.majorTicks), model1.n_epochs)
# Test even
tick_frequency = 2
ax = model1.plot_training_history(tick_frequency=tick_frequency)
self.assertEqual(len(ax.xaxis.majorTicks), model1.n_epochs // tick_frequency + 1)
# Test odd
tick_frequency = 3
ax = model1.plot_training_history(tick_frequency=tick_frequency)
self.assertEqual(len(ax.xaxis.majorTicks), model1.n_epochs // tick_frequency + 1)
# Test max
tick_frequency = model1.n_epochs
ax = model1.plot_training_history(tick_frequency=tick_frequency)
self.assertEqual(len(ax.xaxis.majorTicks), model1.n_epochs // tick_frequency + 1)
# Test 0
tick_frequency = 0
ax = model1.plot_training_history(tick_frequency=tick_frequency)
self.assertEqual(len(ax.xaxis.majorTicks), model1.n_epochs)
if (caslib is not None) and tmp_caslib:
self.s.retrieve('table.dropcaslib', message_level = 'error', caslib = caslib)
def test_model17(self):
# test GlobalAveragePooling
try:
import onnx
except:
unittest.TestCase.skipTest(self, "onnx not found in the libraries")
from dlpy.applications import ResNet18_SAS
model1 = ResNet18_SAS(self.s, model_table='ResNet18')
if self.data_dir is None:
unittest.TestCase.skipTest(
self, "DLPY_DATA_DIR is not set in the environment variables")
caslib, path, tmp_caslib = caslibify(self.s,
path=self.data_dir +
'images.sashdat',
task='load')
self.s.table.loadtable(caslib=caslib,
casout={
'name': 'eee',
'replace': True
},
path=path)
r = model1.fit(data='eee',
inputs='_image_',
target='_label_',
max_epochs=1)
self.assertTrue(r.severity == 0)
import tempfile
tmp_dir_to_dump = tempfile.gettempdir()
model1.deploy(tmp_dir_to_dump, output_format='onnx')
import os
os.remove(os.path.join(tmp_dir_to_dump, "ResNet18.onnx"))
if (caslib is not None) and tmp_caslib:
self.s.retrieve('table.dropcaslib',
message_level='error',
caslib=caslib)
def load_language_model(self, language_model_path):
"""
Load the N-gram language model.
Parameters
----------
language_model_path : string
Specifies the absolute server-side path of the acoustic model file.
"""
self.language_model_caslib, path_after_caslib, _ = caslibify(
self.conn, language_model_path, task="load")
rt = self.conn.retrieve("langModel.lmImport",
_messagelevel='error',
table=dict(name=path_after_caslib,
caslib=self.language_model_caslib),
casout=dict(replace=True,
name=self.language_model_name,
caslib=self.language_model_caslib))
if rt.severity > 1:
self.language_model_caslib = None
for msg in rt.messages:
print(msg)
raise DLPyError("Failed to import the language model.")
def test_transformers4(self):
'''
Load a base BERT model and add classification head.
COVERAGE: BERT_Model() class in bert_model.py
all private class functions (e.g. _XXX) in bert_model.py
compile() in bert_model.py
load_weights() in bert_model.py
write_block_information() in bert_utils.py
get_data_spec() in bert_model.py
create_data_spec() in bert_utils.py
generate_target_var_names() in bert_utils.py
extract_pytorch_parms() in bert_utils.py
find_pytorch_tensor() in bert_utils.py
'''
model_name = 'bert-base-uncased'
cache_dir = self.data_dir_local
try:
stderr = sys.stderr
sys.stderr = open(os.devnull, 'w')
import h5py
h5py_installed = True
sys.stderr = stderr
except:
h5py_installed = False
if self.data_dir is None:
unittest.TestCase.skipTest(
self, "DLPY_DATA_DIR is not set in the environment variables")
if (self.data_dir_local is None) or (not os.path.isdir(
self.data_dir_local)):
unittest.TestCase.skipTest(
self, "DLPY_DATA_DIR_LOCAL is not set in the environment "
"variables or it does not exist.")
if (not self.necessary_packages_installed) or (not h5py_installed):
unittest.TestCase.skipTest(self,
"missing transformers or h5py package")
# test case parameters
n_classes = 2
num_encoder_layers = 2
num_tgt_var = 1
# instantiate BERT model
bert = BERT_Model(self.s,
cache_dir,
model_name,
n_classes,
num_hidden_layers=num_encoder_layers,
verbose=False)
# compile model
bert.compile(num_target_var=num_tgt_var)
if not os.path.isfile(
os.path.join(cache_dir, model_name + '.kerasmodel.h5')):
assertTrue(False, "HDF5 file not written.")
# check for the existence of the model table
res = self.s.retrieve('table.tableexists',
_messagelevel='error',
name=bert.model_name)
self.assertTrue(res['exists'] != 0, "Model table not created.")
# attempt to create CASLIB to cache directory
try:
caslib, extra_path, newlib = caslibify(self.s,
cache_dir,
task='save')
do_load_weights = True
except DLPyError:
do_load_weights = False
# attach model weights - skip if server unable to "see" cache directory
if do_load_weights:
bert.load_weights(os.path.join(cache_dir,
model_name + '.kerasmodel.h5'),
num_target_var=num_tgt_var,
freeze_base_model=False)
# check for the existence of the weight table
res = self.s.retrieve('table.tableexists',
_messagelevel='error',
name=bert.model_name + '_weights')
self.assertTrue(res['exists'] != 0, "Weight table not created.")
# create data spec for model
data_spec = bert.get_data_spec(num_tgt_var)
# drop table(s)
try:
model_tbl_opts = input_table_check(bert.model_name)
self.s.table.droptable(quiet=True, **model_tbl_opts)
except TypeError:
self.assertTrue(False, "Unable to drop model table.")
if do_load_weights:
try:
model_tbl_opts = input_table_check(bert.model_name +
'_weights')
self.s.table.droptable(quiet=True, **model_tbl_opts)
except TypeError:
self.assertTrue(False, "Unable to drop weight table.")
# remove HDF5 file
if os.path.isfile(
os.path.join(cache_dir, model_name + '.kerasmodel.h5')):
os.remove(os.path.join(cache_dir, model_name + '.kerasmodel.h5'))
# clean up BERT model
del bert
def __init__(self,
conn,
data_path,
local_path=None,
acoustic_model_path=None,
language_model_path=None):
try:
import wave
except ImportError:
raise DLPyError(
"wave package was not found. "
"Please install this package before using any APIs from dlpy.speech. "
"We're using this Python library to help read and write audio files."
)
try:
import audioop
except ImportError:
raise DLPyError(
"audioop package was not found. "
"Please install this package before using any APIs from dlpy.speech. "
"We're using this Python library to help extract audio features and convert audio formats."
)
self.conn = conn
self.server_sep = get_server_path_sep(self.conn)
self.data_path = data_path
if self.data_path.endswith(self.server_sep):
self.data_path = self.data_path[:-1]
self.data_path += self.server_sep
server_type = get_cas_host_type(self.conn).lower()
is_server_unix = server_type.startswith(
"lin") or server_type.startswith("osx")
client_type = platform.system()
if (is_server_unix and client_type.startswith("Win")
) or not (is_server_unix or client_type.startswith("Win")):
if local_path is None:
raise DLPyError(
"the \"local_path\" parameter is not specified. "
"The CAS server and the Python client have different OS type (Windows/Linux), "
"so please specify the \"local_path\" parameter.")
else:
self.local_path = local_path
else:
if local_path is None:
self.local_path = self.data_path
print(
"Note: the \"local_path\" parameter is not specified. "
"The CAS server and the Python client have the same OS type (Windows/Linux), "
"so simply use \"data_path\" as \"local_path\":",
self.local_path)
else:
self.local_path = local_path
if not os.path.exists(self.local_path):
raise DLPyError("Invalid \"local_path\" value: does not exist.")
if not os.access(self.local_path, os.R_OK):
raise DLPyError(
"Invalid \"local_path\" value: does not have reading permission."
)
if not os.access(self.local_path, os.W_OK):
raise DLPyError(
"Invalid \"local_path\" value: does not have writing permission."
)
self.conn.loadactionset("audio", _messagelevel="error")
self.conn.loadactionset("deepLearn", _messagelevel="error")
self.conn.loadactionset("langModel", _messagelevel="error")
if acoustic_model_path is not None:
self.load_acoustic_model(acoustic_model_path)
if language_model_path is not None:
self.load_language_model(language_model_path)
self.data_caslib, self.data_path_after_caslib, _ = caslibify(
self.conn, self.data_path, task="save")
self.data_caslib_path = self.conn.caslibinfo(
caslib=self.data_caslib).CASLibInfo["Path"][0]
if not self.data_caslib_path.endswith(self.server_sep):
self.data_caslib_path += self.server_sep
def test_model4(self):
try:
import onnx
from onnx import numpy_helper
except:
unittest.TestCase.skipTest(self, "onnx not found in the libraries")
import numpy as np
model1 = Sequential(self.s, model_table='Simple_CNN1')
model1.add(InputLayer(3, 224, 224))
model1.add(Conv2d(8, 7, act='identity', include_bias=False))
model1.add(Reshape(height=448, width=448, depth=2, act='IDENTITY'))
model1.add(Reshape(height=448, width=448, depth=2, act='RECTIFIER'))
model1.add(Conv2d(8, 7, act='identity', include_bias=False))
model1.add(BN(act='relu'))
model1.add(Dense(2))
model1.add(OutputLayer(act='softmax', n=2))
if self.data_dir is None:
unittest.TestCase.skipTest(
self, "DLPY_DATA_DIR is not set in the environment variables")
caslib, path, tmp_caslib = caslibify(self.s,
path=self.data_dir +
'images.sashdat',
task='load')
self.s.table.loadtable(caslib=caslib,
casout={
'name': 'eee',
'replace': True
},
path=path)
r = model1.fit(data='eee',
inputs='_image_',
target='_label_',
max_epochs=1)
self.assertTrue(r.severity == 0)
if self.data_dir_local is None:
unittest.TestCase.skipTest(
self,
"DLPY_DATA_DIR_LOCAL is not set in the environment variables")
#model1.deploy(self.data_dir_local, output_format='onnx')
import tempfile
tmp_dir_to_dump = tempfile.gettempdir()
model1.deploy(tmp_dir_to_dump, output_format='onnx')
import os
model_path = os.path.join(tmp_dir_to_dump, 'Simple_CNN1.onnx')
m = onnx.load(model_path)
self.assertEqual(m.graph.node[1].op_type, 'Reshape')
init = numpy_helper.to_array(m.graph.initializer[1])
self.assertTrue(np.array_equal(init, [-1, 2, 448, 448]))
import os
os.remove(os.path.join(tmp_dir_to_dump, "Simple_CNN1.onnx"))
if (caslib is not None) and tmp_caslib:
self.s.retrieve('table.dropcaslib',
message_level='error',
caslib=caslib)
