def set_optimizer_parameters(self,
beta1=0.9,
beta2=0.999,
learning_rate=3e-5,
max_epochs=3,
mini_batch_size=1):
'''
Sets optimizer parameters according to defaults in original BERT paper.
Parameters
----------
beta1 : double, optional
Specifies the exponential decay rate for the first moment in
the Adam learning algorithm.
Default: 0.9
beta2 : double, optional
Specifies the exponential decay rate for the second moment in
the Adam learning algorithm.
Default: 0.999
learning_rate : double, optional
Specifies the learning rate.
Default: 3e-5
max_epochs : int, optional
Specifies the maximum number of epochs.
Default: 3
mini_batch_size : int, optional
Specifies the number of observations per thread in a mini-batch.
Default: 1
'''
# the optimizer dictionary is set according to the parameter choices recommended
# for fine-tuning in the original BERT paper
if self._verbose:
log_level = 3
else:
log_level = 0
# determine what to train
if self._freeze_base:
freeze_layers_to = self._rnn_layer
freeze_layers = None
else:
freeze_layers_to = None
freeze_layers = [self._rnn_layer]
# set optimizer parameters similar to settings in original BERT paper
self._optimizer = Optimizer(algorithm=AdamSolver(
beta1=beta1,
beta2=beta2,
learning_rate=learning_rate,
gamma=0.0,
power=0.0),
mini_batch_size=mini_batch_size,
seed=self.seed,
max_epochs=max_epochs,
dropout_type='inverted',
log_level=log_level,
freeze_layers_to=freeze_layers_to,
freeze_layers=freeze_layers)
def test_CyclicLR(self):
model1 = Sequential(self.s, model_table = 'Simple_CNN1')
model1.add(InputLayer(3, 224, 224))
model1.add(Conv2d(8, 7))
model1.add(Pooling(2))
model1.add(Conv2d(8, 7))
model1.add(Pooling(2))
model1.add(Dense(16))
model1.add(OutputLayer(act = 'softmax', n = 2))
if self.data_dir is None:
unittest.TestCase.skipTest(self, "DLPY_DATA_DIR is not set in the environment variables")
caslib, path, tmp_caslib = caslibify(self.s, path = self.data_dir + 'images.sashdat', task = 'load')
self.s.table.loadtable(caslib = caslib,
casout = {'name': 'eee', 'replace': True},
path = path)
lrs = CyclicLR(self.s, 'eee', 4, 1.0, 0.0000001, 0.01)
solver = VanillaSolver(lr_scheduler=lrs)
self.assertTrue(self.sample_syntax['CyclicLR'] == solver)
optimizer = Optimizer(algorithm = solver, log_level = 3, max_epochs = 4, mini_batch_size = 2)
r = model1.fit(data = 'eee', inputs = '_image_', target = '_label_', optimizer = optimizer, n_threads=2)
if r.severity > 0:
for msg in r.messages:
print(msg)
self.assertTrue(r.severity <= 1)
def test_siamese_fit_1(self):
if self.server_dir is None:
unittest.TestCase.skipTest(
self,
"DLPY_DATA_DIR_SERVER is not set in the environment variables")
# test using one data table
resnet18_model = ResNet18_Caffe(self.s,
width=224,
height=224,
random_crop='RESIZETHENCROP',
random_flip='HV',
random_mutation='random')
embedding_layer = Dense(n=4)
model1 = EmbeddingModel.build_embedding_model(
resnet18_model,
model_table='test1',
embedding_model_type='siamese',
margin=3.0,
embedding_layer=embedding_layer)
res1 = model1.print_summary()
self.assertEqual(
res1[res1['Layer'].str.contains(
model1.embedding_layer_name_prefix)].shape[0], 2)
img_path = self.server_dir + 'DogBreed_small'
my_images = ImageEmbeddingTable.load_files(
self.s,
path=img_path,
n_samples=64,
embedding_model_type='siamese')
solver = AdamSolver(lr_scheduler=StepLR(learning_rate=0.0001,
step_size=20),
clip_grad_max=100,
clip_grad_min=-100)
optimizer = Optimizer(algorithm=solver,
mini_batch_size=8,
log_level=3,
max_epochs=2,
reg_l2=0.0001)
gpu = Gpu(devices=1)
train_res = model1.fit_embedding_model(optimizer=optimizer,
n_threads=1,
gpu=gpu,
path=img_path,
n_samples=64,
max_iter=2,
seed=1234,
record_seed=23435)
print(train_res)
score_res = model1.predict(data=my_images,
gpu=gpu,
random_crop='RESIZETHENCROP')
print(score_res)
def test_build_gan_model_4(self):
if self.server_dir is None:
unittest.TestCase.skipTest(self, "DLPY_DATA_DIR_SERVER is not set in the environment variables")
discriminator = Sequential(self.s)
discriminator.add(InputLayer(1, 28, 28))
discriminator.add(Conv2d(3, 3))
discriminator.add(Pooling(2))
discriminator.add(Conv2d(3, 3))
discriminator.add(Pooling(2))
discriminator.add(Dense(16))
discriminator.add(OutputLayer(n=1))
generator = Sequential(self.s)
generator.add(InputLayer(1, 100, 1))
generator.add(Dense(256, act='relu'))
generator.add(Dense(512, act='relu'))
generator.add(Dense(1024, act='relu'))
generator.add(Dense(28 * 28, act='tanh'))
generator.add(OutputLayer(act='softmax', n=2))
encoder = Sequential(self.s)
encoder.add(InputLayer(100, 1, 1))
encoder.add(Dense(256, act='relu'))
encoder.add(Dense(512, act='relu'))
encoder.add(Dense(1024, act='relu'))
encoder.add(Dense(100, act='tanh'))
encoder.add(OutputLayer(act='softmax', n=2))
gan_model = GANModel(generator, discriminator, encoder)
res = gan_model.models['generator'].print_summary()
print(res)
res = gan_model.models['discriminator'].print_summary()
print(res)
from dlpy.model import Optimizer, MomentumSolver, AdamSolver
solver = AdamSolver(lr_scheduler=StepLR(learning_rate=0.0001, step_size=4), clip_grad_max=100,
clip_grad_min=-100)
optimizer = Optimizer(algorithm=solver, mini_batch_size=8, log_level=2, max_epochs=4, reg_l2=0.0001)
res = gan_model.fit(optimizer, optimizer, self.server_dir + 'mnist_validate',
n_samples_generator=32, n_samples_discriminator=32, max_iter=2, n_threads=1,
damping_factor=0.5)
print(res)
def test_model_forecast3(self):
import datetime
try:
import pandas as pd
except:
unittest.TestCase.skipTest(self, "pandas not found in the libraries")
import numpy as np
filename1 = os.path.join(os.path.dirname(__file__), 'datasources', 'timeseries_exp1.csv')
importoptions1 = dict(filetype='delimited', delimiter=',')
if self.data_dir is None:
unittest.TestCase.skipTest(self, "DLPY_DATA_DIR is not set in the environment variables")
self.table3 = TimeseriesTable.from_localfile(self.s, filename1, importoptions=importoptions1)
self.table3.timeseries_formatting(timeid='datetime',
timeseries=['series', 'covar'],
timeid_informat='ANYDTDTM19.',
timeid_format='DATETIME19.')
self.table3.timeseries_accumlation(acc_interval='day',
groupby=['id1var', 'id2var'])
self.table3.prepare_subsequences(seq_len=2,
target='series',
predictor_timeseries=['series', 'covar'],
missing_handling='drop')
valid_start = datetime.date(2015, 1, 4)
test_start = datetime.date(2015, 1, 7)
traintbl, validtbl, testtbl = self.table3.timeseries_partition(
validation_start=valid_start, testing_start=test_start)
sascode = '''
data {};
set {};
drop series_lag1;
run;
'''.format(validtbl.name, validtbl.name)
self.s.retrieve('dataStep.runCode', _messagelevel='error', code=sascode)
sascode = '''
data {};
set {};
drop series_lag1;
run;
'''.format(testtbl.name, testtbl.name)
self.s.retrieve('dataStep.runCode', _messagelevel='error', code=sascode)
model1 = Sequential(self.s, model_table='lstm_rnn')
model1.add(InputLayer(std='STD'))
model1.add(Recurrent(rnn_type='LSTM', output_type='encoding', n=15, reversed_=False))
model1.add(OutputLayer(act='IDENTITY'))
optimizer = Optimizer(algorithm=AdamSolver(learning_rate=0.01), mini_batch_size=32,
seed=1234, max_epochs=10)
seq_spec = Sequence(**traintbl.sequence_opt)
result = model1.fit(traintbl, optimizer=optimizer,
sequence=seq_spec, **traintbl.inputs_target)
self.assertTrue(result.severity == 0)
resulttbl1 = model1.forecast(validtbl, horizon=1)
self.assertTrue(isinstance(resulttbl1, CASTable))
self.assertTrue(resulttbl1.shape[0]==15)
local_resulttbl1 = resulttbl1.to_frame()
unique_time = local_resulttbl1.datetime.unique()
self.assertTrue(len(unique_time)==1)
self.assertTrue(pd.Timestamp(unique_time[0])==datetime.datetime(2015,1,4))
resulttbl2 = model1.forecast(validtbl, horizon=3)
self.assertTrue(isinstance(resulttbl2, CASTable))
self.assertTrue(resulttbl2.shape[0]==45)
local_resulttbl2 = resulttbl2.to_frame()
local_resulttbl2.sort_values(by=['id1var', 'id2var', 'datetime'], inplace=True)
unique_time = local_resulttbl2.datetime.unique()
self.assertTrue(len(unique_time)==3)
for i in range(3):
self.assertTrue(pd.Timestamp(unique_time[i])==datetime.datetime(2015,1,4+i))
series_lag1 = local_resulttbl2.loc[(local_resulttbl2.id1var==1) & (local_resulttbl2.id2var==1),
'series_lag1'].values
series_lag2 = local_resulttbl2.loc[(local_resulttbl2.id1var==1) & (local_resulttbl2.id2var==1),
'series_lag2'].values
DL_Pred = local_resulttbl2.loc[(local_resulttbl2.id1var==1) & (local_resulttbl2.id2var==1),
'_DL_Pred_'].values
self.assertTrue(np.array_equal(series_lag1[1:3], DL_Pred[0:2]))
self.assertTrue(series_lag2[2]==DL_Pred[0])
with self.assertRaises(RuntimeError):
resulttbl3 = model1.forecast(testtbl, horizon=3)
def test_quartet_fit(self):
if self.server_dir is None:
unittest.TestCase.skipTest(
self,
"DLPY_DATA_DIR_SERVER is not set in the environment variables")
# test using one data table
resnet18_model = ResNet18_Caffe(self.s,
width=224,
height=224,
random_crop='RESIZETHENCROP',
random_flip='HV',
random_mutation='random')
embedding_layer = Dense(n=4)
model1 = EmbeddingModel.build_embedding_model(
resnet18_model,
model_table='test1',
embedding_model_type='quartet',
margin=-3.0,
embedding_layer=embedding_layer)
res1 = model1.print_summary()
print(res1)
img_path = self.server_dir + 'DogBreed_small'
my_images = ImageEmbeddingTable.load_files(
self.s,
path=img_path,
n_samples=64,
embedding_model_type='quartet')
solver = AdamSolver(lr_scheduler=StepLR(learning_rate=0.0001,
step_size=20),
clip_grad_max=100,
clip_grad_min=-100)
optimizer = Optimizer(algorithm=solver,
mini_batch_size=8,
log_level=3,
max_epochs=5,
reg_l2=0.0001)
gpu = Gpu(devices=1)
train_res = model1.fit_embedding_model(data=my_images,
optimizer=optimizer,
n_threads=1,
gpu=gpu,
seed=1234,
record_seed=23435)
print(train_res)
score_res = model1.predict(data=my_images,
gpu=gpu,
random_crop='RESIZETHENCROP')
print(score_res)
# test deploy as astore
self.s.loadactionset('astore')
my_images_test = ImageEmbeddingTable.load_files(
self.s,
path=img_path,
n_samples=5,
embedding_model_type='quartet',
resize_width=224,
resize_height=224)
# case 1: deploy the full model as astore
model1.deploy_embedding_model(output_format='astore',
model_type='full',
path=self.local_dir)
full_astore = os.path.join(self.local_dir,
model1.model_name + '.astore')
with open(full_astore, mode='rb') as file:
file_content = file.read()
store_ = swat.blob(file_content)
self.s.astore.upload(rstore=dict(name='test1_full', replace=True),
store=store_)
# run with one gpu
self.s.score(
rstore=dict(name='test1_full'),
table=my_images_test,
nthreads=1,
# _debug=dict(ranks=0),
copyvars=['_path_', '_path_1', '_path_2', '_path_3'],
options=[
dict(name='usegpu', value='1'),
dict(name='NDEVICES', value='1'),
dict(name='DEVICE0', value='0')
],
out=dict(name='astore_score1_full_gpu', replace=True))
res = self.s.fetch(table='astore_score1_full_gpu')
print(res)
# remove the astore file
os.remove(full_astore)
# case 2: deploy the branch model as astore
model1.deploy_embedding_model(output_format='astore',
model_type='branch',
path=self.local_dir)
br_astore = os.path.join(self.local_dir,
model1.model_name + '_branch.astore')
with open(br_astore, mode='rb') as file:
file_content = file.read()
store_ = swat.blob(file_content)
self.s.astore.upload(rstore=dict(name='test1_br', replace=True),
store=store_)
# run with one gpu
self.s.score(
rstore=dict(name='test1_br'),
table=my_images_test,
nthreads=1,
# _debug=dict(ranks=0),
copyvars=['_path_'],
options=[
dict(name='usegpu', value='1'),
dict(name='NDEVICES', value='1'),
dict(name='DEVICE0', value='0')
],
out=dict(name='astore_score1_br_gpu', replace=True))
res = self.s.fetch(table='astore_score1_br_gpu')
print(res)
os.remove(br_astore)