def test_siamese_fit_1(self):
if self.server_dir is None:
unittest.TestCase.skipTest(
self,
"DLPY_DATA_DIR_SERVER is not set in the environment variables")
# test using one data table
resnet18_model = ResNet18_Caffe(self.s,
width=224,
height=224,
random_crop='RESIZETHENCROP',
random_flip='HV',
random_mutation='random')
embedding_layer = Dense(n=4)
model1 = EmbeddingModel.build_embedding_model(
resnet18_model,
model_table='test1',
embedding_model_type='siamese',
margin=3.0,
embedding_layer=embedding_layer)
res1 = model1.print_summary()
self.assertEqual(
res1[res1['Layer'].str.contains(
model1.embedding_layer_name_prefix)].shape[0], 2)
img_path = self.server_dir + 'DogBreed_small'
my_images = ImageEmbeddingTable.load_files(
self.s,
path=img_path,
n_samples=64,
embedding_model_type='siamese')
solver = AdamSolver(lr_scheduler=StepLR(learning_rate=0.0001,
step_size=20),
clip_grad_max=100,
clip_grad_min=-100)
optimizer = Optimizer(algorithm=solver,
mini_batch_size=8,
log_level=3,
max_epochs=2,
reg_l2=0.0001)
gpu = Gpu(devices=1)
train_res = model1.fit_embedding_model(optimizer=optimizer,
n_threads=1,
gpu=gpu,
path=img_path,
n_samples=64,
max_iter=2,
seed=1234,
record_seed=23435)
print(train_res)
score_res = model1.predict(data=my_images,
gpu=gpu,
random_crop='RESIZETHENCROP')
print(score_res)
def test_build_gan_model_4(self):
if self.server_dir is None:
unittest.TestCase.skipTest(self, "DLPY_DATA_DIR_SERVER is not set in the environment variables")
discriminator = Sequential(self.s)
discriminator.add(InputLayer(1, 28, 28))
discriminator.add(Conv2d(3, 3))
discriminator.add(Pooling(2))
discriminator.add(Conv2d(3, 3))
discriminator.add(Pooling(2))
discriminator.add(Dense(16))
discriminator.add(OutputLayer(n=1))
generator = Sequential(self.s)
generator.add(InputLayer(1, 100, 1))
generator.add(Dense(256, act='relu'))
generator.add(Dense(512, act='relu'))
generator.add(Dense(1024, act='relu'))
generator.add(Dense(28 * 28, act='tanh'))
generator.add(OutputLayer(act='softmax', n=2))
encoder = Sequential(self.s)
encoder.add(InputLayer(100, 1, 1))
encoder.add(Dense(256, act='relu'))
encoder.add(Dense(512, act='relu'))
encoder.add(Dense(1024, act='relu'))
encoder.add(Dense(100, act='tanh'))
encoder.add(OutputLayer(act='softmax', n=2))
gan_model = GANModel(generator, discriminator, encoder)
res = gan_model.models['generator'].print_summary()
print(res)
res = gan_model.models['discriminator'].print_summary()
print(res)
from dlpy.model import Optimizer, MomentumSolver, AdamSolver
solver = AdamSolver(lr_scheduler=StepLR(learning_rate=0.0001, step_size=4), clip_grad_max=100,
clip_grad_min=-100)
optimizer = Optimizer(algorithm=solver, mini_batch_size=8, log_level=2, max_epochs=4, reg_l2=0.0001)
res = gan_model.fit(optimizer, optimizer, self.server_dir + 'mnist_validate',
n_samples_generator=32, n_samples_discriminator=32, max_iter=2, n_threads=1,
damping_factor=0.5)
print(res)
def test_quartet_fit(self):
if self.server_dir is None:
unittest.TestCase.skipTest(
self,
"DLPY_DATA_DIR_SERVER is not set in the environment variables")
# test using one data table
resnet18_model = ResNet18_Caffe(self.s,
width=224,
height=224,
random_crop='RESIZETHENCROP',
random_flip='HV',
random_mutation='random')
embedding_layer = Dense(n=4)
model1 = EmbeddingModel.build_embedding_model(
resnet18_model,
model_table='test1',
embedding_model_type='quartet',
margin=-3.0,
embedding_layer=embedding_layer)
res1 = model1.print_summary()
print(res1)
img_path = self.server_dir + 'DogBreed_small'
my_images = ImageEmbeddingTable.load_files(
self.s,
path=img_path,
n_samples=64,
embedding_model_type='quartet')
solver = AdamSolver(lr_scheduler=StepLR(learning_rate=0.0001,
step_size=20),
clip_grad_max=100,
clip_grad_min=-100)
optimizer = Optimizer(algorithm=solver,
mini_batch_size=8,
log_level=3,
max_epochs=5,
reg_l2=0.0001)
gpu = Gpu(devices=1)
train_res = model1.fit_embedding_model(data=my_images,
optimizer=optimizer,
n_threads=1,
gpu=gpu,
seed=1234,
record_seed=23435)
print(train_res)
score_res = model1.predict(data=my_images,
gpu=gpu,
random_crop='RESIZETHENCROP')
print(score_res)
# test deploy as astore
self.s.loadactionset('astore')
my_images_test = ImageEmbeddingTable.load_files(
self.s,
path=img_path,
n_samples=5,
embedding_model_type='quartet',
resize_width=224,
resize_height=224)
# case 1: deploy the full model as astore
model1.deploy_embedding_model(output_format='astore',
model_type='full',
path=self.local_dir)
full_astore = os.path.join(self.local_dir,
model1.model_name + '.astore')
with open(full_astore, mode='rb') as file:
file_content = file.read()
store_ = swat.blob(file_content)
self.s.astore.upload(rstore=dict(name='test1_full', replace=True),
store=store_)
# run with one gpu
self.s.score(
rstore=dict(name='test1_full'),
table=my_images_test,
nthreads=1,
# _debug=dict(ranks=0),
copyvars=['_path_', '_path_1', '_path_2', '_path_3'],
options=[
dict(name='usegpu', value='1'),
dict(name='NDEVICES', value='1'),
dict(name='DEVICE0', value='0')
],
out=dict(name='astore_score1_full_gpu', replace=True))
res = self.s.fetch(table='astore_score1_full_gpu')
print(res)
# remove the astore file
os.remove(full_astore)
# case 2: deploy the branch model as astore
model1.deploy_embedding_model(output_format='astore',
model_type='branch',
path=self.local_dir)
br_astore = os.path.join(self.local_dir,
model1.model_name + '_branch.astore')
with open(br_astore, mode='rb') as file:
file_content = file.read()
store_ = swat.blob(file_content)
self.s.astore.upload(rstore=dict(name='test1_br', replace=True),
store=store_)
# run with one gpu
self.s.score(
rstore=dict(name='test1_br'),
table=my_images_test,
nthreads=1,
# _debug=dict(ranks=0),
copyvars=['_path_'],
options=[
dict(name='usegpu', value='1'),
dict(name='NDEVICES', value='1'),
dict(name='DEVICE0', value='0')
],
out=dict(name='astore_score1_br_gpu', replace=True))
res = self.s.fetch(table='astore_score1_br_gpu')
print(res)
os.remove(br_astore)
def test_StepLR(self):
lr_policy = StepLR(1.0, 1.2, 30)
self.assertTrue(self.sample_syntax['StepLR'] == lr_policy)
def test_compatiable_syntax(self):
VanillaSolver(lr_scheduler=StepLR(1.0, 1.2, 30),
learning_rate=0.001,
learning_rate_policy='fixed')