def test_load_files_resize(self):
if self.server_dir is None:
unittest.TestCase.skipTest(
self,
"DLPY_DATA_DIR_SERVER is not set in the environment variables")
img_path = self.server_dir + 'DogBreed_small'
my_images = ImageEmbeddingTable.load_files(self.s,
path=img_path,
n_samples=128,
resize_width=224,
resize_height=224)
print(my_images.columns)
print(my_images.head())
out_freq = my_images.freq(inputs='_dissimilar_')
out_freq = out_freq['Frequency']
print(out_freq)
label, label1, label_pair, dissimilar = my_images.label_freq
print(label)
print(label1)
print(label_pair)
print(dissimilar)
my_images.show(randomize=True, n_image_pairs=10)
self.assertTrue(len(my_images) > 0)
self.assertEqual(dissimilar['Frequency'][0], out_freq['Frequency'][0])
def test_siamese_fit_1(self):
if self.server_dir is None:
unittest.TestCase.skipTest(
self,
"DLPY_DATA_DIR_SERVER is not set in the environment variables")
# test using one data table
resnet18_model = ResNet18_Caffe(self.s,
width=224,
height=224,
random_crop='RESIZETHENCROP',
random_flip='HV',
random_mutation='random')
embedding_layer = Dense(n=4)
model1 = EmbeddingModel.build_embedding_model(
resnet18_model,
model_table='test1',
embedding_model_type='siamese',
margin=3.0,
embedding_layer=embedding_layer)
res1 = model1.print_summary()
self.assertEqual(
res1[res1['Layer'].str.contains(
model1.embedding_layer_name_prefix)].shape[0], 2)
img_path = self.server_dir + 'DogBreed_small'
my_images = ImageEmbeddingTable.load_files(
self.s,
path=img_path,
n_samples=64,
embedding_model_type='siamese')
solver = AdamSolver(lr_scheduler=StepLR(learning_rate=0.0001,
step_size=20),
clip_grad_max=100,
clip_grad_min=-100)
optimizer = Optimizer(algorithm=solver,
mini_batch_size=8,
log_level=3,
max_epochs=2,
reg_l2=0.0001)
gpu = Gpu(devices=1)
train_res = model1.fit_embedding_model(optimizer=optimizer,
n_threads=1,
gpu=gpu,
path=img_path,
n_samples=64,
max_iter=2,
seed=1234,
record_seed=23435)
print(train_res)
score_res = model1.predict(data=my_images,
gpu=gpu,
random_crop='RESIZETHENCROP')
print(score_res)
def test_load_files_triplet(self):
if self.server_dir is None:
unittest.TestCase.skipTest(self, "DLPY_DATA_DIR_SERVER is not set in the environment variables")
img_path = self.server_dir + 'DogBreed_small'
my_images = ImageEmbeddingTable.load_files(self.s, path=img_path, embedding_model_type='triplet')
print(my_images.columns)
print(my_images.head())
label, label1, label2, label_triplet = my_images.label_freq
print(label)
print(label1)
print(label2)
print(label_triplet)
my_images.show(randomize=True, n_image_pairs=2)
self.assertTrue(len(my_images) > 0)
self.assertTrue(label_triplet['Frequency'][0] > 0)
def test_quartet_fit(self):
if self.server_dir is None:
unittest.TestCase.skipTest(
self,
"DLPY_DATA_DIR_SERVER is not set in the environment variables")
# test using one data table
resnet18_model = ResNet18_Caffe(self.s,
width=224,
height=224,
random_crop='RESIZETHENCROP',
random_flip='HV',
random_mutation='random')
embedding_layer = Dense(n=4)
model1 = EmbeddingModel.build_embedding_model(
resnet18_model,
model_table='test1',
embedding_model_type='quartet',
margin=-3.0,
embedding_layer=embedding_layer)
res1 = model1.print_summary()
print(res1)
img_path = self.server_dir + 'DogBreed_small'
my_images = ImageEmbeddingTable.load_files(
self.s,
path=img_path,
n_samples=64,
embedding_model_type='quartet')
solver = AdamSolver(lr_scheduler=StepLR(learning_rate=0.0001,
step_size=20),
clip_grad_max=100,
clip_grad_min=-100)
optimizer = Optimizer(algorithm=solver,
mini_batch_size=8,
log_level=3,
max_epochs=5,
reg_l2=0.0001)
gpu = Gpu(devices=1)
train_res = model1.fit_embedding_model(data=my_images,
optimizer=optimizer,
n_threads=1,
gpu=gpu,
seed=1234,
record_seed=23435)
print(train_res)
score_res = model1.predict(data=my_images,
gpu=gpu,
random_crop='RESIZETHENCROP')
print(score_res)
# test deploy as astore
self.s.loadactionset('astore')
my_images_test = ImageEmbeddingTable.load_files(
self.s,
path=img_path,
n_samples=5,
embedding_model_type='quartet',
resize_width=224,
resize_height=224)
# case 1: deploy the full model as astore
model1.deploy_embedding_model(output_format='astore',
model_type='full',
path=self.local_dir)
full_astore = os.path.join(self.local_dir,
model1.model_name + '.astore')
with open(full_astore, mode='rb') as file:
file_content = file.read()
store_ = swat.blob(file_content)
self.s.astore.upload(rstore=dict(name='test1_full', replace=True),
store=store_)
# run with one gpu
self.s.score(
rstore=dict(name='test1_full'),
table=my_images_test,
nthreads=1,
# _debug=dict(ranks=0),
copyvars=['_path_', '_path_1', '_path_2', '_path_3'],
options=[
dict(name='usegpu', value='1'),
dict(name='NDEVICES', value='1'),
dict(name='DEVICE0', value='0')
],
out=dict(name='astore_score1_full_gpu', replace=True))
res = self.s.fetch(table='astore_score1_full_gpu')
print(res)
# remove the astore file
os.remove(full_astore)
# case 2: deploy the branch model as astore
model1.deploy_embedding_model(output_format='astore',
model_type='branch',
path=self.local_dir)
br_astore = os.path.join(self.local_dir,
model1.model_name + '_branch.astore')
with open(br_astore, mode='rb') as file:
file_content = file.read()
store_ = swat.blob(file_content)
self.s.astore.upload(rstore=dict(name='test1_br', replace=True),
store=store_)
# run with one gpu
self.s.score(
rstore=dict(name='test1_br'),
table=my_images_test,
nthreads=1,
# _debug=dict(ranks=0),
copyvars=['_path_'],
options=[
dict(name='usegpu', value='1'),
dict(name='NDEVICES', value='1'),
dict(name='DEVICE0', value='0')
],
out=dict(name='astore_score1_br_gpu', replace=True))
res = self.s.fetch(table='astore_score1_br_gpu')
print(res)
os.remove(br_astore)