def test_model_conversion3(self): ''' Import CNN image classification model and override attributes - instantiate a Keras LeNet model and translate to DLPy/Viya model override CNN model attributes with RNN atttributes - never would be done in practice, just to verify that new attributes written NOTE: cannot attach weights unless both client and server share the same file system COVERAGE: from_keras_model(), load_weights() in network.py keras_to_sas() in sas_keras_parse.py write_keras_hdf5() in write_keras_model_parm.py all functions in model_conversion_utils.py CNN-related function in write_sas_code.py ''' if self.data_dir is None: unittest.TestCase.skipTest( self, "DLPY_DATA_DIR is not set in the environment variables") if (self.data_dir_local is None) or (not os.path.isfile( os.path.join(self.data_dir_local, 'lenet.h5'))): unittest.TestCase.skipTest( self, "DLPY_DATA_DIR_LOCAL is not set in the environment variables or lenet.h5 file is missing" ) if self.keras_installed: from keras.models import Sequential from keras.layers import Dense, Dropout, Conv2D, MaxPooling2D, Flatten else: unittest.TestCase.skipTest(self, "keras is not installed") model = Sequential() model.add( Conv2D(20, kernel_size=(5, 5), strides=(1, 1), activation='relu', input_shape=(28, 28, 1), padding="same")) model.add( MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same')) model.add( Conv2D(50, kernel_size=(5, 5), strides=(1, 1), activation='relu', padding='same')) model.add( MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same')) model.add(Flatten()) model.add(Dense(500, activation='relu')) model.add(Dense(10, activation='softmax')) model.load_weights(os.path.join(self.data_dir_local, 'lenet.h5')) model.summary() model_name = 'lenet' model1, use_gpu = Model.from_keras_model( conn=self.s, keras_model=model, output_model_table=model_name, include_weights=True, scale=1.0 / 255.0, input_weights_file=os.path.join(self.data_dir_local, 'lenet.h5')) if os.path.isdir(self.data_dir): try: copyfile( os.path.join(os.getcwd(), 'lenet_weights.kerasmodel.h5'), os.path.join(self.data_dir, 'lenet_weights.kerasmodel.h5')) copy_success = True except: print( 'Unable to copy weights file, skipping test of overriding attributes' ) copy_success = False if copy_success: self.s.table.addcaslib(activeonadd=False, datasource={'srctype': 'path'}, name='MODEL_CONVERT', path=self.data_dir, subdirectories=True) model1.load_weights(path=os.path.join( self.data_dir, 'lenet_weights.kerasmodel.h5'), labels=False, use_gpu=use_gpu) os.remove( os.path.join(self.data_dir, 'lenet_weights.kerasmodel.h5')) # parameter for (nonexistent) RNN layers rnn_size = 10 feature_dim = 4 # output classes output_dim = 29 # maximum sequence length max_seq_len = 100 # define data specs needed to import Keras model weights tokensize = feature_dim inputs = [] for fi in range(max_seq_len): for vi in range(tokensize): inputs.append('_f%d_v%d_' % (fi, vi)) targets = ['y%d' % i for i in range(0, max_seq_len)] data_spec = [] data_spec.append( DataSpec(type_='NUMERICNOMINAL', layer=model.layers[0].name + "_input", data=inputs, numeric_nominal_parms=DataSpecNumNomOpts( length='_num_frames_', token_size=feature_dim))) data_spec.append( DataSpec(type_='NUMERICNOMINAL', layer=model.layers[-1].name, data=targets, nominals=targets, numeric_nominal_parms=DataSpecNumNomOpts( length='ylen', token_size=1))) # override model attributes from dlpy.attribute_utils import create_extended_attributes create_extended_attributes(self.s, model_name, model1.layers, data_spec) if os.path.isfile( os.path.join(os.getcwd(), 'lenet_weights.kerasmodel.h5')): os.remove(os.path.join(os.getcwd(), 'lenet_weights.kerasmodel.h5')) # clean up model table model_tbl_opts = input_table_check(model_name) self.s.table.droptable(quiet=True, **model_tbl_opts) # clean up models del model del model1
def test_model_conversion1(self): ''' Import CNN image classification model - instantiate a Keras LeNet model and translate to DLPy/Viya model NOTE: cannot attach weights unless both client and server share the same file system COVERAGE: from_keras_model(), load_weights() in network.py keras_to_sas() in sas_keras_parse.py write_keras_hdf5_from_file() in write_keras_model_parm.py all functions in model_conversion_utils.py all function in keras_utils.py CNN-related function in write_sas_code.py ''' if self.data_dir is None: unittest.TestCase.skipTest( self, "DLPY_DATA_DIR is not set in the environment variables") if (self.data_dir_local is None) or (not os.path.isfile( os.path.join(self.data_dir_local, 'lenet.h5'))): unittest.TestCase.skipTest( self, "DLPY_DATA_DIR_LOCAL is not set in the environment " "variables or lenet.h5 file is missing") if self.keras_installed: from keras.models import Sequential from keras.layers import Dense, Dropout, Conv2D, MaxPooling2D, Flatten else: unittest.TestCase.skipTest(self, "keras is not installed") model = Sequential() model.add( Conv2D(20, kernel_size=(5, 5), strides=(1, 1), activation='relu', input_shape=(28, 28, 1), padding="same")) model.add( MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same')) model.add( Conv2D(50, kernel_size=(5, 5), strides=(1, 1), activation='relu', padding='same')) model.add( MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same')) model.add(Flatten()) model.add(Dense(500, activation='relu')) model.add(Dense(10, activation='softmax')) model.load_weights(os.path.join(self.data_dir_local, 'lenet.h5')) model.summary() model_name = 'lenet' model1, use_gpu = Model.from_keras_model( conn=self.s, keras_model=model, output_model_table=model_name, include_weights=True, scale=1.0 / 255.0, input_weights_file=os.path.join(self.data_dir_local, 'lenet.h5')) if os.path.isdir(self.data_dir): try: copyfile( os.path.join(os.getcwd(), 'lenet_weights.kerasmodel.h5'), os.path.join(self.data_dir, 'lenet_weights.kerasmodel.h5')) copy_success = True except: print( 'Unable to copy weights file, skipping test of attaching weights' ) copy_success = False if copy_success: model1.load_weights(path=os.path.join( self.data_dir, 'lenet_weights.kerasmodel.h5'), labels=False, use_gpu=use_gpu) os.remove( os.path.join(self.data_dir, 'lenet_weights.kerasmodel.h5')) model1.print_summary() if os.path.isfile( os.path.join(os.getcwd(), 'lenet_weights.kerasmodel.h5')): os.remove(os.path.join(os.getcwd(), 'lenet_weights.kerasmodel.h5')) # clean up model table model_tbl_opts = input_table_check(model_name) self.s.table.droptable(quiet=True, **model_tbl_opts) # clean up models del model del model1
def test_model_conversion2(self): ''' Import RNN sequence to sequence models - instantiate Keras RNN models and translate to DLPy/Viya models NOTE: cannot attach weights unless both client and server share the same file system COVERAGE: from_keras_model(), load_weights() in network.py keras_to_sas() in sas_keras_parse.py write_keras_hdf5() in write_keras_model_parm.py all functions in model_conversion_utils.py CNN-related function in write_sas_code.py ''' if self.data_dir is None: unittest.TestCase.skipTest( self, "DLPY_DATA_DIR is not set in the environment variables") if (self.data_dir_local is None) or (not os.path.isfile( os.path.join(self.data_dir_local, 'lenet.h5'))): unittest.TestCase.skipTest( self, "DLPY_DATA_DIR_LOCAL is not set in the environment variables or lenet.h5 file is missing" ) if not self.keras_installed: unittest.TestCase.skipTest(self, "keras is not installed") # parameter for RNN layers rnn_size = 10 feature_dim = 4 # output classes output_dim = 29 # maximum sequence length max_seq_len = 100 # define data specs needed to import Keras model weights tokensize = feature_dim inputs = [] for fi in range(max_seq_len): for vi in range(tokensize): inputs.append('_f%d_v%d_' % (fi, vi)) targets = ['y%d' % i for i in range(0, max_seq_len)] data_spec = [] data_spec.append( DataSpec(type_='NUMERICNOMINAL', layer='the_input', data=inputs, numeric_nominal_parms=DataSpecNumNomOpts( length='_num_frames_', token_size=feature_dim))) data_spec.append( DataSpec(type_='NUMERICNOMINAL', layer='out', data=targets, nominals=targets, numeric_nominal_parms=DataSpecNumNomOpts(length='ylen', token_size=1))) # try all RNN model types for layer_type in [ 'simplernn', 'lstm', 'gru', 'cudnnlstm', 'cudnngru' ]: for bidirectional in [True, False]: model = define_keras_rnn_model(layer_type, bidirectional, rnn_size, feature_dim, output_dim) model_name = 'dlpy_model' model1, use_gpu = Model.from_keras_model( conn=self.s, keras_model=model, max_num_frames=max_seq_len, include_weights=True, output_model_table=model_name) model1.print_summary() # try to load weights, but skip any GPU-based models because worker/soloist may not have GPU if os.path.isdir(self.data_dir) and (not use_gpu): try: copyfile( os.path.join(os.getcwd(), 'dlpy_model_weights.kerasmodel.h5'), os.path.join(self.data_dir, 'dlpy_model_weights.kerasmodel.h5')) copy_success = True except: print( 'Unable to copy weights file, skipping test of attaching weights' ) copy_success = False if copy_success: model1.load_weights(path=os.path.join( self.data_dir, 'dlpy_model_weights.kerasmodel.h5'), labels=False, use_gpu=use_gpu) os.remove( os.path.join(self.data_dir, 'dlpy_model_weights.kerasmodel.h5')) else: print('GPU model, skipping test of attaching weights') if os.path.isfile( os.path.join(os.getcwd(), 'dlpy_model_weights.kerasmodel.h5')): os.remove( os.path.join(os.getcwd(), 'dlpy_model_weights.kerasmodel.h5')) # clean up models del model del model1 # clean up model table model_tbl_opts = input_table_check(model_name) self.s.table.droptable(quiet=True, **model_tbl_opts)
def test_model_conversion1(self): if self.data_dir is None: unittest.TestCase.skipTest( self, "DLPY_DATA_DIR is not set in the environment variables") if (self.data_dir_local is None) or (not os.path.isfile( os.path.join(self.data_dir_local, 'lenet.h5'))): unittest.TestCase.skipTest( self, "DLPY_DATA_DIR_LOCAL is not set in the environment variables or lenet.h5 file is missing" ) model = Sequential() model.add( Conv2D(20, kernel_size=(5, 5), strides=(1, 1), activation='relu', input_shape=(28, 28, 1), padding="same")) model.add( MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same')) model.add( Conv2D(50, kernel_size=(5, 5), strides=(1, 1), activation='relu', padding='same')) model.add( MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding='same')) model.add(Flatten()) model.add(Dense(500, activation='relu')) model.add(Dense(10, activation='softmax')) model.load_weights(os.path.join(self.data_dir_local, 'lenet.h5')) model.summary() model_name = 'lenet' model1 = Model.from_keras_model(conn=self.s, keras_model=model, output_model_table=model_name, include_weights=True, scale=1.0 / 255.0, input_weights_file=os.path.join( self.data_dir_local, 'lenet.h5')) if os.path.isdir(self.data_dir): try: copyfile( os.path.join(os.getcwd(), 'lenet_weights.kerasmodel.h5'), os.path.join(self.data_dir, 'lenet_weights.kerasmodel.h5')) copy_success = True except: print( 'Unable to copy weights file, skipping test of attaching weights' ) copy_success = False if copy_success: model1.load_weights(path=os.path.join( self.data_dir, 'lenet_weights.kerasmodel.h5'), labels=False) os.remove( os.path.join(self.data_dir, 'lenet_weights.kerasmodel.h5')) model1.print_summary() if os.path.isfile( os.path.join(os.getcwd(), 'lenet_weights.kerasmodel.h5')): os.remove(os.path.join(os.getcwd(), 'lenet_weights.kerasmodel.h5'))