def test_model7(self):
model1 = Sequential(self.s, model_table='Simple_CNN1')
model1.add(InputLayer(3, 224, 224))
model1.add(Conv2d(8, 7))
model1.add(Pooling(2))
model1.add(Conv2d(8, 7))
model1.add(Pooling(2))
model1.add(Dense(16))
model1.add(OutputLayer(act='softmax', n=2))
if self.data_dir is None:
unittest.TestCase.skipTest(
self, "DLPY_DATA_DIR is not set in the environment variables")
caslib, path = caslibify(self.s,
path=self.data_dir + 'images.sashdat',
task='load')
self.s.table.loadtable(caslib=caslib,
casout={
'name': 'eee',
'replace': True
},
path=path)
r = model1.fit(data='eee',
inputs='_image_',
target='_label_',
save_best_weights=True)
self.assertTrue(r.severity == 0)
r2 = model1.predict(data='eee', use_best_weights=True)
self.assertTrue(r2.severity == 0)
def test_model15(self):
# test RECTIFIER activation for concat layer
try:
import onnx
except:
unittest.TestCase.skipTest(self, "onnx not found in the libraries")
model1 = Sequential(self.s, model_table='Simple_CNN1')
model1.add(InputLayer(3, 224, 224))
model1.add(Conv2d(8, 7))
pool1 = Pooling(2)
model1.add(pool1)
conv1 = Conv2d(1, 7, src_layers=[pool1])
conv2 = Conv2d(1, 7, src_layers=[pool1])
model1.add(conv1)
model1.add(conv2)
model1.add(Concat(act='RECTIFIER', src_layers=[conv1, conv2]))
model1.add(Pooling(2))
model1.add(Dense(2))
model1.add(OutputLayer(act='softmax', n=2))
if self.data_dir is None:
unittest.TestCase.skipTest(
self, "DLPY_DATA_DIR is not set in the environment variables")
caslib, path, tmp_caslib = caslibify(self.s,
path=self.data_dir +
'images.sashdat',
task='load')
self.s.table.loadtable(caslib=caslib,
casout={
'name': 'eee',
'replace': True
},
path=path)
r = model1.fit(data='eee',
inputs='_image_',
target='_label_',
max_epochs=1)
self.assertTrue(r.severity == 0)
import tempfile
tmp_dir_to_dump = tempfile.gettempdir()
model1.deploy(tmp_dir_to_dump, output_format='onnx')
import os
os.remove(os.path.join(tmp_dir_to_dump, "Simple_CNN1.onnx"))
if (caslib is not None) and tmp_caslib:
self.s.retrieve('table.dropcaslib',
message_level='error',
caslib=caslib)
def test_model13(self):
# test dropout
try:
import onnx
except:
unittest.TestCase.skipTest(self, "onnx not found in the libraries")
model1 = Sequential(self.s, model_table='Simple_CNN1')
model1.add(InputLayer(3, 224, 224))
model1.add(
Conv2d(8, 7, act='IDENTITY', dropout=0.5, include_bias=False))
model1.add(BN(act='relu'))
model1.add(Pooling(2, pool='MEAN', dropout=0.5))
model1.add(
Conv2d(8, 7, act='IDENTITY', dropout=0.5, include_bias=False))
model1.add(BN(act='relu'))
model1.add(Pooling(2, pool='MEAN', dropout=0.5))
model1.add(Conv2d(8, 7, act='identity', include_bias=False))
model1.add(BN(act='relu'))
model1.add(Dense(16, act='IDENTITY', dropout=0.1))
model1.add(OutputLayer(act='softmax', n=2))
if self.data_dir is None:
unittest.TestCase.skipTest(
self, "DLPY_DATA_DIR is not set in the environment variables")
caslib, path, tmp_caslib = caslibify(self.s,
path=self.data_dir +
'images.sashdat',
task='load')
self.s.table.loadtable(caslib=caslib,
casout={
'name': 'eee',
'replace': True
},
path=path)
r = model1.fit(data='eee',
inputs='_image_',
target='_label_',
max_epochs=2)
self.assertTrue(r.severity == 0)
import tempfile
tmp_dir_to_dump = tempfile.gettempdir()
model1.deploy(tmp_dir_to_dump, output_format='onnx')
import os
os.remove(os.path.join(tmp_dir_to_dump, "Simple_CNN1.onnx"))
if (caslib is not None) and tmp_caslib:
self.s.retrieve('table.dropcaslib',
message_level='error',
caslib=caslib)
def test_imagescaler2(self):
# test export model with imagescaler
try:
import onnx
except:
unittest.TestCase.skipTest(self, 'onnx not found')
if self.data_dir_local is None:
unittest.TestCase.skipTest(
self, 'DLPY_DATA_DIR_LOCAL is not set in '
'the environment variables')
model1 = Sequential(self.s, model_table='imagescaler2')
model1.add(
InputLayer(n_channels=3,
width=224,
height=224,
scale=1 / 255.,
offsets=[0.1, 0.2, 0.3]))
model1.add(Conv2d(8, 7))
model1.add(Pooling(2))
model1.add(OutputLayer(act='softmax', n=2))
caslib, path = caslibify(self.s,
path=self.data_dir + 'images.sashdat',
task='load')
self.s.table.loadtable(caslib=caslib,
casout={
'name': 'eee',
'replace': True
},
path=path)
r = model1.fit(data='eee',
inputs='_image_',
target='_label_',
max_epochs=1)
self.assertTrue(r.severity == 0)
from dlpy.model_conversion.write_onnx_model import sas_to_onnx
onnx_model = sas_to_onnx(model1.layers,
self.s.CASTable('imagescaler2'),
self.s.CASTable('imagescaler2_weights'))
self.assertAlmostEqual(onnx_model.graph.node[0].attribute[0].floats[0],
0.1)
self.assertAlmostEqual(onnx_model.graph.node[0].attribute[0].floats[1],
0.2)
self.assertAlmostEqual(onnx_model.graph.node[0].attribute[0].floats[2],
0.3)
self.assertAlmostEqual(onnx_model.graph.node[0].attribute[1].f,
1 / 255.)
def test_plot_ticks(self):
model1 = Sequential(self.s, model_table='Simple_CNN1')
model1.add(InputLayer(3, 224, 224))
model1.add(Conv2d(8, 7))
model1.add(Pooling(2))
model1.add(Conv2d(8, 7))
model1.add(Pooling(2))
model1.add(Dense(16))
model1.add(OutputLayer(act='softmax', n=2))
if self.data_dir is None:
unittest.TestCase.skipTest(self, "DLPY_DATA_DIR is not set in the environment variables")
caslib, path, tmp_caslib = caslibify(self.s, path=self.data_dir+'images.sashdat', task='load')
self.s.table.loadtable(caslib=caslib,
casout={'name': 'eee', 'replace': True},
path=path)
r = model1.fit(data='eee', inputs='_image_', target='_label_', lr=0.001, max_epochs=5)
# Test default tick_frequency value of 1
ax = model1.plot_training_history()
self.assertEqual(len(ax.xaxis.majorTicks), model1.n_epochs)
# Test even
tick_frequency = 2
ax = model1.plot_training_history(tick_frequency=tick_frequency)
self.assertEqual(len(ax.xaxis.majorTicks), model1.n_epochs // tick_frequency + 1)
# Test odd
tick_frequency = 3
ax = model1.plot_training_history(tick_frequency=tick_frequency)
self.assertEqual(len(ax.xaxis.majorTicks), model1.n_epochs // tick_frequency + 1)
# Test max
tick_frequency = model1.n_epochs
ax = model1.plot_training_history(tick_frequency=tick_frequency)
self.assertEqual(len(ax.xaxis.majorTicks), model1.n_epochs // tick_frequency + 1)
# Test 0
tick_frequency = 0
ax = model1.plot_training_history(tick_frequency=tick_frequency)
self.assertEqual(len(ax.xaxis.majorTicks), model1.n_epochs)
if (caslib is not None) and tmp_caslib:
self.s.retrieve('table.dropcaslib', message_level = 'error', caslib = caslib)
def test_model21(self):
try:
import onnx
except:
unittest.TestCase.skipTest(self, "onnx not found in the libraries")
model1 = Sequential(self.s, model_table='Simple_CNN1')
model1.add(InputLayer(3, 224, 224))
model1.add(Conv2d(8, 7))
pool1 = Pooling(2)
model1.add(pool1)
conv1 = Conv2d(1, 7, src_layers=[pool1])
conv2 = Conv2d(1, 7, src_layers=[pool1])
model1.add(conv1)
model1.add(conv2)
model1.add(Concat(act='identity', src_layers=[conv1, conv2]))
model1.add(Pooling(2))
model1.add(Dense(2))
model1.add(OutputLayer(act='softmax', n=2))
if self.data_dir is None:
unittest.TestCase.skipTest(
self, "DLPY_DATA_DIR is not set in the environment variables")
caslib, path = caslibify(self.s,
path=self.data_dir + 'images.sashdat',
task='load')
self.s.table.loadtable(caslib=caslib,
casout={
'name': 'eee',
'replace': True
},
path=path)
r = model1.fit(data='eee',
inputs='_image_',
target='_label_',
max_epochs=1)
self.assertTrue(r.severity == 0)
model1.deploy(self.data_dir, output_format='onnx')
def test_model20(self):
try:
import onnx
except:
unittest.TestCase.skipTest(self, "onnx not found in the libraries")
model1 = Sequential(self.s, model_table='Simple_CNN1')
model1.add(InputLayer(3, 224, 224))
model1.add(Conv2d(8, 7))
model1.add(Pooling(2))
model1.add(Conv2d(8, 7))
model1.add(Pooling(2))
model1.add(Dense(16))
model1.add(OutputLayer(act='softmax', n=2))
if self.data_dir is None:
unittest.TestCase.skipTest(
self, "DLPY_DATA_DIR is not set in the environment variables")
caslib, path = caslibify(self.s,
path=self.data_dir + 'images.sashdat',
task='load')
self.s.table.loadtable(caslib=caslib,
casout={
'name': 'eee',
'replace': True
},
path=path)
r = model1.fit(data='eee',
inputs='_image_',
target='_label_',
max_epochs=1)
self.assertTrue(r.severity == 0)
model1.save_weights_csv(self.data_dir)
weights_path = os.path.join(self.data_dir_local,
'Simple_CNN1_weights.csv')
model1.deploy(self.data_dir_local,
output_format='onnx',
model_weights=weights_path)
def test_CyclicLR(self):
model1 = Sequential(self.s, model_table='Simple_CNN1')
model1.add(InputLayer(3, 224, 224))
model1.add(Conv2d(8, 7))
model1.add(Pooling(2))
model1.add(Conv2d(8, 7))
model1.add(Pooling(2))
model1.add(Dense(16))
model1.add(OutputLayer(act='softmax', n=2))
if self.data_dir is None:
unittest.TestCase.skipTest(
self, "DLPY_DATA_DIR is not set in the environment variables")
caslib, path, tmp_caslib = caslibify(self.s,
path=self.data_dir +
'images.sashdat',
task='load')
self.s.table.loadtable(caslib=caslib,
casout={
'name': 'eee',
'replace': True
},
path=path)
lrs = CyclicLR(self.s, 'eee', 4, 1.0, 0.0000001, 0.01)
solver = VanillaSolver(lr_scheduler=lrs)
self.assertTrue(self.sample_syntax['CyclicLR'] == solver)
optimizer = Optimizer(algorithm=solver,
log_level=3,
max_epochs=4,
mini_batch_size=2)
r = model1.fit(data='eee',
inputs='_image_',
target='_label_',
optimizer=optimizer,
n_threads=2)
if r.severity > 0:
for msg in r.messages:
print(msg)
self.assertTrue(r.severity <= 1)
def test_model_forecast3(self):
import datetime
try:
import pandas as pd
except:
unittest.TestCase.skipTest(self, "pandas not found in the libraries")
import numpy as np
filename1 = os.path.join(os.path.dirname(__file__), 'datasources', 'timeseries_exp1.csv')
importoptions1 = dict(filetype='delimited', delimiter=',')
if self.data_dir is None:
unittest.TestCase.skipTest(self, "DLPY_DATA_DIR is not set in the environment variables")
self.table3 = TimeseriesTable.from_localfile(self.s, filename1, importoptions=importoptions1)
self.table3.timeseries_formatting(timeid='datetime',
timeseries=['series', 'covar'],
timeid_informat='ANYDTDTM19.',
timeid_format='DATETIME19.')
self.table3.timeseries_accumlation(acc_interval='day',
groupby=['id1var', 'id2var'])
self.table3.prepare_subsequences(seq_len=2,
target='series',
predictor_timeseries=['series', 'covar'],
missing_handling='drop')
valid_start = datetime.date(2015, 1, 4)
test_start = datetime.date(2015, 1, 7)
traintbl, validtbl, testtbl = self.table3.timeseries_partition(
validation_start=valid_start, testing_start=test_start)
sascode = '''
data {};
set {};
drop series_lag1;
run;
'''.format(validtbl.name, validtbl.name)
self.s.retrieve('dataStep.runCode', _messagelevel='error', code=sascode)
sascode = '''
data {};
set {};
drop series_lag1;
run;
'''.format(testtbl.name, testtbl.name)
self.s.retrieve('dataStep.runCode', _messagelevel='error', code=sascode)
model1 = Sequential(self.s, model_table='lstm_rnn')
model1.add(InputLayer(std='STD'))
model1.add(Recurrent(rnn_type='LSTM', output_type='encoding', n=15, reversed_=False))
model1.add(OutputLayer(act='IDENTITY'))
optimizer = Optimizer(algorithm=AdamSolver(learning_rate=0.01), mini_batch_size=32,
seed=1234, max_epochs=10)
seq_spec = Sequence(**traintbl.sequence_opt)
result = model1.fit(traintbl, optimizer=optimizer,
sequence=seq_spec, **traintbl.inputs_target)
self.assertTrue(result.severity == 0)
resulttbl1 = model1.forecast(validtbl, horizon=1)
self.assertTrue(isinstance(resulttbl1, CASTable))
self.assertTrue(resulttbl1.shape[0]==15)
local_resulttbl1 = resulttbl1.to_frame()
unique_time = local_resulttbl1.datetime.unique()
self.assertTrue(len(unique_time)==1)
self.assertTrue(pd.Timestamp(unique_time[0])==datetime.datetime(2015,1,4))
resulttbl2 = model1.forecast(validtbl, horizon=3)
self.assertTrue(isinstance(resulttbl2, CASTable))
self.assertTrue(resulttbl2.shape[0]==45)
local_resulttbl2 = resulttbl2.to_frame()
local_resulttbl2.sort_values(by=['id1var', 'id2var', 'datetime'], inplace=True)
unique_time = local_resulttbl2.datetime.unique()
self.assertTrue(len(unique_time)==3)
for i in range(3):
self.assertTrue(pd.Timestamp(unique_time[i])==datetime.datetime(2015,1,4+i))
series_lag1 = local_resulttbl2.loc[(local_resulttbl2.id1var==1) & (local_resulttbl2.id2var==1),
'series_lag1'].values
series_lag2 = local_resulttbl2.loc[(local_resulttbl2.id1var==1) & (local_resulttbl2.id2var==1),
'series_lag2'].values
DL_Pred = local_resulttbl2.loc[(local_resulttbl2.id1var==1) & (local_resulttbl2.id2var==1),
'_DL_Pred_'].values
self.assertTrue(np.array_equal(series_lag1[1:3], DL_Pred[0:2]))
self.assertTrue(series_lag2[2]==DL_Pred[0])
with self.assertRaises(RuntimeError):
resulttbl3 = model1.forecast(testtbl, horizon=3)
def test_model4(self):
try:
import onnx
from onnx import numpy_helper
except:
unittest.TestCase.skipTest(self, "onnx not found in the libraries")
import numpy as np
model1 = Sequential(self.s, model_table='Simple_CNN1')
model1.add(InputLayer(3, 224, 224))
model1.add(Conv2d(8, 7, act='identity', include_bias=False))
model1.add(Reshape(height=448, width=448, depth=2, act='IDENTITY'))
model1.add(Reshape(height=448, width=448, depth=2, act='RECTIFIER'))
model1.add(Conv2d(8, 7, act='identity', include_bias=False))
model1.add(BN(act='relu'))
model1.add(Dense(2))
model1.add(OutputLayer(act='softmax', n=2))
if self.data_dir is None:
unittest.TestCase.skipTest(
self, "DLPY_DATA_DIR is not set in the environment variables")
caslib, path, tmp_caslib = caslibify(self.s,
path=self.data_dir +
'images.sashdat',
task='load')
self.s.table.loadtable(caslib=caslib,
casout={
'name': 'eee',
'replace': True
},
path=path)
r = model1.fit(data='eee',
inputs='_image_',
target='_label_',
max_epochs=1)
self.assertTrue(r.severity == 0)
if self.data_dir_local is None:
unittest.TestCase.skipTest(
self,
"DLPY_DATA_DIR_LOCAL is not set in the environment variables")
#model1.deploy(self.data_dir_local, output_format='onnx')
import tempfile
tmp_dir_to_dump = tempfile.gettempdir()
model1.deploy(tmp_dir_to_dump, output_format='onnx')
import os
model_path = os.path.join(tmp_dir_to_dump, 'Simple_CNN1.onnx')
m = onnx.load(model_path)
self.assertEqual(m.graph.node[1].op_type, 'Reshape')
init = numpy_helper.to_array(m.graph.initializer[1])
self.assertTrue(np.array_equal(init, [-1, 2, 448, 448]))
import os
os.remove(os.path.join(tmp_dir_to_dump, "Simple_CNN1.onnx"))
if (caslib is not None) and tmp_caslib:
self.s.retrieve('table.dropcaslib',
message_level='error',
caslib=caslib)
