def test_model7(self): model1 = Sequential(self.s, model_table='Simple_CNN1') model1.add(InputLayer(3, 224, 224)) model1.add(Conv2d(8, 7)) model1.add(Pooling(2)) model1.add(Conv2d(8, 7)) model1.add(Pooling(2)) model1.add(Dense(16)) model1.add(OutputLayer(act='softmax', n=2)) if self.data_dir is None: unittest.TestCase.skipTest( self, "DLPY_DATA_DIR is not set in the environment variables") caslib, path = caslibify(self.s, path=self.data_dir + 'images.sashdat', task='load') self.s.table.loadtable(caslib=caslib, casout={ 'name': 'eee', 'replace': True }, path=path) r = model1.fit(data='eee', inputs='_image_', target='_label_', save_best_weights=True) self.assertTrue(r.severity == 0) r2 = model1.predict(data='eee', use_best_weights=True) self.assertTrue(r2.severity == 0)
def test_model15(self): # test RECTIFIER activation for concat layer try: import onnx except: unittest.TestCase.skipTest(self, "onnx not found in the libraries") model1 = Sequential(self.s, model_table='Simple_CNN1') model1.add(InputLayer(3, 224, 224)) model1.add(Conv2d(8, 7)) pool1 = Pooling(2) model1.add(pool1) conv1 = Conv2d(1, 7, src_layers=[pool1]) conv2 = Conv2d(1, 7, src_layers=[pool1]) model1.add(conv1) model1.add(conv2) model1.add(Concat(act='RECTIFIER', src_layers=[conv1, conv2])) model1.add(Pooling(2)) model1.add(Dense(2)) model1.add(OutputLayer(act='softmax', n=2)) if self.data_dir is None: unittest.TestCase.skipTest( self, "DLPY_DATA_DIR is not set in the environment variables") caslib, path, tmp_caslib = caslibify(self.s, path=self.data_dir + 'images.sashdat', task='load') self.s.table.loadtable(caslib=caslib, casout={ 'name': 'eee', 'replace': True }, path=path) r = model1.fit(data='eee', inputs='_image_', target='_label_', max_epochs=1) self.assertTrue(r.severity == 0) import tempfile tmp_dir_to_dump = tempfile.gettempdir() model1.deploy(tmp_dir_to_dump, output_format='onnx') import os os.remove(os.path.join(tmp_dir_to_dump, "Simple_CNN1.onnx")) if (caslib is not None) and tmp_caslib: self.s.retrieve('table.dropcaslib', message_level='error', caslib=caslib)
def test_model13(self): # test dropout try: import onnx except: unittest.TestCase.skipTest(self, "onnx not found in the libraries") model1 = Sequential(self.s, model_table='Simple_CNN1') model1.add(InputLayer(3, 224, 224)) model1.add( Conv2d(8, 7, act='IDENTITY', dropout=0.5, include_bias=False)) model1.add(BN(act='relu')) model1.add(Pooling(2, pool='MEAN', dropout=0.5)) model1.add( Conv2d(8, 7, act='IDENTITY', dropout=0.5, include_bias=False)) model1.add(BN(act='relu')) model1.add(Pooling(2, pool='MEAN', dropout=0.5)) model1.add(Conv2d(8, 7, act='identity', include_bias=False)) model1.add(BN(act='relu')) model1.add(Dense(16, act='IDENTITY', dropout=0.1)) model1.add(OutputLayer(act='softmax', n=2)) if self.data_dir is None: unittest.TestCase.skipTest( self, "DLPY_DATA_DIR is not set in the environment variables") caslib, path, tmp_caslib = caslibify(self.s, path=self.data_dir + 'images.sashdat', task='load') self.s.table.loadtable(caslib=caslib, casout={ 'name': 'eee', 'replace': True }, path=path) r = model1.fit(data='eee', inputs='_image_', target='_label_', max_epochs=2) self.assertTrue(r.severity == 0) import tempfile tmp_dir_to_dump = tempfile.gettempdir() model1.deploy(tmp_dir_to_dump, output_format='onnx') import os os.remove(os.path.join(tmp_dir_to_dump, "Simple_CNN1.onnx")) if (caslib is not None) and tmp_caslib: self.s.retrieve('table.dropcaslib', message_level='error', caslib=caslib)
def test_imagescaler2(self): # test export model with imagescaler try: import onnx except: unittest.TestCase.skipTest(self, 'onnx not found') if self.data_dir_local is None: unittest.TestCase.skipTest( self, 'DLPY_DATA_DIR_LOCAL is not set in ' 'the environment variables') model1 = Sequential(self.s, model_table='imagescaler2') model1.add( InputLayer(n_channels=3, width=224, height=224, scale=1 / 255., offsets=[0.1, 0.2, 0.3])) model1.add(Conv2d(8, 7)) model1.add(Pooling(2)) model1.add(OutputLayer(act='softmax', n=2)) caslib, path = caslibify(self.s, path=self.data_dir + 'images.sashdat', task='load') self.s.table.loadtable(caslib=caslib, casout={ 'name': 'eee', 'replace': True }, path=path) r = model1.fit(data='eee', inputs='_image_', target='_label_', max_epochs=1) self.assertTrue(r.severity == 0) from dlpy.model_conversion.write_onnx_model import sas_to_onnx onnx_model = sas_to_onnx(model1.layers, self.s.CASTable('imagescaler2'), self.s.CASTable('imagescaler2_weights')) self.assertAlmostEqual(onnx_model.graph.node[0].attribute[0].floats[0], 0.1) self.assertAlmostEqual(onnx_model.graph.node[0].attribute[0].floats[1], 0.2) self.assertAlmostEqual(onnx_model.graph.node[0].attribute[0].floats[2], 0.3) self.assertAlmostEqual(onnx_model.graph.node[0].attribute[1].f, 1 / 255.)
def test_plot_ticks(self): model1 = Sequential(self.s, model_table='Simple_CNN1') model1.add(InputLayer(3, 224, 224)) model1.add(Conv2d(8, 7)) model1.add(Pooling(2)) model1.add(Conv2d(8, 7)) model1.add(Pooling(2)) model1.add(Dense(16)) model1.add(OutputLayer(act='softmax', n=2)) if self.data_dir is None: unittest.TestCase.skipTest(self, "DLPY_DATA_DIR is not set in the environment variables") caslib, path, tmp_caslib = caslibify(self.s, path=self.data_dir+'images.sashdat', task='load') self.s.table.loadtable(caslib=caslib, casout={'name': 'eee', 'replace': True}, path=path) r = model1.fit(data='eee', inputs='_image_', target='_label_', lr=0.001, max_epochs=5) # Test default tick_frequency value of 1 ax = model1.plot_training_history() self.assertEqual(len(ax.xaxis.majorTicks), model1.n_epochs) # Test even tick_frequency = 2 ax = model1.plot_training_history(tick_frequency=tick_frequency) self.assertEqual(len(ax.xaxis.majorTicks), model1.n_epochs // tick_frequency + 1) # Test odd tick_frequency = 3 ax = model1.plot_training_history(tick_frequency=tick_frequency) self.assertEqual(len(ax.xaxis.majorTicks), model1.n_epochs // tick_frequency + 1) # Test max tick_frequency = model1.n_epochs ax = model1.plot_training_history(tick_frequency=tick_frequency) self.assertEqual(len(ax.xaxis.majorTicks), model1.n_epochs // tick_frequency + 1) # Test 0 tick_frequency = 0 ax = model1.plot_training_history(tick_frequency=tick_frequency) self.assertEqual(len(ax.xaxis.majorTicks), model1.n_epochs) if (caslib is not None) and tmp_caslib: self.s.retrieve('table.dropcaslib', message_level = 'error', caslib = caslib)
def test_model21(self): try: import onnx except: unittest.TestCase.skipTest(self, "onnx not found in the libraries") model1 = Sequential(self.s, model_table='Simple_CNN1') model1.add(InputLayer(3, 224, 224)) model1.add(Conv2d(8, 7)) pool1 = Pooling(2) model1.add(pool1) conv1 = Conv2d(1, 7, src_layers=[pool1]) conv2 = Conv2d(1, 7, src_layers=[pool1]) model1.add(conv1) model1.add(conv2) model1.add(Concat(act='identity', src_layers=[conv1, conv2])) model1.add(Pooling(2)) model1.add(Dense(2)) model1.add(OutputLayer(act='softmax', n=2)) if self.data_dir is None: unittest.TestCase.skipTest( self, "DLPY_DATA_DIR is not set in the environment variables") caslib, path = caslibify(self.s, path=self.data_dir + 'images.sashdat', task='load') self.s.table.loadtable(caslib=caslib, casout={ 'name': 'eee', 'replace': True }, path=path) r = model1.fit(data='eee', inputs='_image_', target='_label_', max_epochs=1) self.assertTrue(r.severity == 0) model1.deploy(self.data_dir, output_format='onnx')
def test_model20(self): try: import onnx except: unittest.TestCase.skipTest(self, "onnx not found in the libraries") model1 = Sequential(self.s, model_table='Simple_CNN1') model1.add(InputLayer(3, 224, 224)) model1.add(Conv2d(8, 7)) model1.add(Pooling(2)) model1.add(Conv2d(8, 7)) model1.add(Pooling(2)) model1.add(Dense(16)) model1.add(OutputLayer(act='softmax', n=2)) if self.data_dir is None: unittest.TestCase.skipTest( self, "DLPY_DATA_DIR is not set in the environment variables") caslib, path = caslibify(self.s, path=self.data_dir + 'images.sashdat', task='load') self.s.table.loadtable(caslib=caslib, casout={ 'name': 'eee', 'replace': True }, path=path) r = model1.fit(data='eee', inputs='_image_', target='_label_', max_epochs=1) self.assertTrue(r.severity == 0) model1.save_weights_csv(self.data_dir) weights_path = os.path.join(self.data_dir_local, 'Simple_CNN1_weights.csv') model1.deploy(self.data_dir_local, output_format='onnx', model_weights=weights_path)
def test_CyclicLR(self): model1 = Sequential(self.s, model_table='Simple_CNN1') model1.add(InputLayer(3, 224, 224)) model1.add(Conv2d(8, 7)) model1.add(Pooling(2)) model1.add(Conv2d(8, 7)) model1.add(Pooling(2)) model1.add(Dense(16)) model1.add(OutputLayer(act='softmax', n=2)) if self.data_dir is None: unittest.TestCase.skipTest( self, "DLPY_DATA_DIR is not set in the environment variables") caslib, path, tmp_caslib = caslibify(self.s, path=self.data_dir + 'images.sashdat', task='load') self.s.table.loadtable(caslib=caslib, casout={ 'name': 'eee', 'replace': True }, path=path) lrs = CyclicLR(self.s, 'eee', 4, 1.0, 0.0000001, 0.01) solver = VanillaSolver(lr_scheduler=lrs) self.assertTrue(self.sample_syntax['CyclicLR'] == solver) optimizer = Optimizer(algorithm=solver, log_level=3, max_epochs=4, mini_batch_size=2) r = model1.fit(data='eee', inputs='_image_', target='_label_', optimizer=optimizer, n_threads=2) if r.severity > 0: for msg in r.messages: print(msg) self.assertTrue(r.severity <= 1)
def test_model_forecast3(self): import datetime try: import pandas as pd except: unittest.TestCase.skipTest(self, "pandas not found in the libraries") import numpy as np filename1 = os.path.join(os.path.dirname(__file__), 'datasources', 'timeseries_exp1.csv') importoptions1 = dict(filetype='delimited', delimiter=',') if self.data_dir is None: unittest.TestCase.skipTest(self, "DLPY_DATA_DIR is not set in the environment variables") self.table3 = TimeseriesTable.from_localfile(self.s, filename1, importoptions=importoptions1) self.table3.timeseries_formatting(timeid='datetime', timeseries=['series', 'covar'], timeid_informat='ANYDTDTM19.', timeid_format='DATETIME19.') self.table3.timeseries_accumlation(acc_interval='day', groupby=['id1var', 'id2var']) self.table3.prepare_subsequences(seq_len=2, target='series', predictor_timeseries=['series', 'covar'], missing_handling='drop') valid_start = datetime.date(2015, 1, 4) test_start = datetime.date(2015, 1, 7) traintbl, validtbl, testtbl = self.table3.timeseries_partition( validation_start=valid_start, testing_start=test_start) sascode = ''' data {}; set {}; drop series_lag1; run; '''.format(validtbl.name, validtbl.name) self.s.retrieve('dataStep.runCode', _messagelevel='error', code=sascode) sascode = ''' data {}; set {}; drop series_lag1; run; '''.format(testtbl.name, testtbl.name) self.s.retrieve('dataStep.runCode', _messagelevel='error', code=sascode) model1 = Sequential(self.s, model_table='lstm_rnn') model1.add(InputLayer(std='STD')) model1.add(Recurrent(rnn_type='LSTM', output_type='encoding', n=15, reversed_=False)) model1.add(OutputLayer(act='IDENTITY')) optimizer = Optimizer(algorithm=AdamSolver(learning_rate=0.01), mini_batch_size=32, seed=1234, max_epochs=10) seq_spec = Sequence(**traintbl.sequence_opt) result = model1.fit(traintbl, optimizer=optimizer, sequence=seq_spec, **traintbl.inputs_target) self.assertTrue(result.severity == 0) resulttbl1 = model1.forecast(validtbl, horizon=1) self.assertTrue(isinstance(resulttbl1, CASTable)) self.assertTrue(resulttbl1.shape[0]==15) local_resulttbl1 = resulttbl1.to_frame() unique_time = local_resulttbl1.datetime.unique() self.assertTrue(len(unique_time)==1) self.assertTrue(pd.Timestamp(unique_time[0])==datetime.datetime(2015,1,4)) resulttbl2 = model1.forecast(validtbl, horizon=3) self.assertTrue(isinstance(resulttbl2, CASTable)) self.assertTrue(resulttbl2.shape[0]==45) local_resulttbl2 = resulttbl2.to_frame() local_resulttbl2.sort_values(by=['id1var', 'id2var', 'datetime'], inplace=True) unique_time = local_resulttbl2.datetime.unique() self.assertTrue(len(unique_time)==3) for i in range(3): self.assertTrue(pd.Timestamp(unique_time[i])==datetime.datetime(2015,1,4+i)) series_lag1 = local_resulttbl2.loc[(local_resulttbl2.id1var==1) & (local_resulttbl2.id2var==1), 'series_lag1'].values series_lag2 = local_resulttbl2.loc[(local_resulttbl2.id1var==1) & (local_resulttbl2.id2var==1), 'series_lag2'].values DL_Pred = local_resulttbl2.loc[(local_resulttbl2.id1var==1) & (local_resulttbl2.id2var==1), '_DL_Pred_'].values self.assertTrue(np.array_equal(series_lag1[1:3], DL_Pred[0:2])) self.assertTrue(series_lag2[2]==DL_Pred[0]) with self.assertRaises(RuntimeError): resulttbl3 = model1.forecast(testtbl, horizon=3)
def test_model4(self): try: import onnx from onnx import numpy_helper except: unittest.TestCase.skipTest(self, "onnx not found in the libraries") import numpy as np model1 = Sequential(self.s, model_table='Simple_CNN1') model1.add(InputLayer(3, 224, 224)) model1.add(Conv2d(8, 7, act='identity', include_bias=False)) model1.add(Reshape(height=448, width=448, depth=2, act='IDENTITY')) model1.add(Reshape(height=448, width=448, depth=2, act='RECTIFIER')) model1.add(Conv2d(8, 7, act='identity', include_bias=False)) model1.add(BN(act='relu')) model1.add(Dense(2)) model1.add(OutputLayer(act='softmax', n=2)) if self.data_dir is None: unittest.TestCase.skipTest( self, "DLPY_DATA_DIR is not set in the environment variables") caslib, path, tmp_caslib = caslibify(self.s, path=self.data_dir + 'images.sashdat', task='load') self.s.table.loadtable(caslib=caslib, casout={ 'name': 'eee', 'replace': True }, path=path) r = model1.fit(data='eee', inputs='_image_', target='_label_', max_epochs=1) self.assertTrue(r.severity == 0) if self.data_dir_local is None: unittest.TestCase.skipTest( self, "DLPY_DATA_DIR_LOCAL is not set in the environment variables") #model1.deploy(self.data_dir_local, output_format='onnx') import tempfile tmp_dir_to_dump = tempfile.gettempdir() model1.deploy(tmp_dir_to_dump, output_format='onnx') import os model_path = os.path.join(tmp_dir_to_dump, 'Simple_CNN1.onnx') m = onnx.load(model_path) self.assertEqual(m.graph.node[1].op_type, 'Reshape') init = numpy_helper.to_array(m.graph.initializer[1]) self.assertTrue(np.array_equal(init, [-1, 2, 448, 448])) import os os.remove(os.path.join(tmp_dir_to_dump, "Simple_CNN1.onnx")) if (caslib is not None) and tmp_caslib: self.s.retrieve('table.dropcaslib', message_level='error', caslib=caslib)