def upload_astore(conn, path, table_name=None):
'''
Function to load the local astore file to server
Parameters:
----------
conn : a cas connection.
path : str
Specifies the full path of the astore file.
table_name : str
Specifies the name of the cas table on server, to hold the astore object.
Return:
----------
None
'''
import swat as sw
conn.loadactionset('astore')
with open(path, 'br') as f:
astore_byte = f.read()
store_ = sw.blob(astore_byte)
if table_name is None:
table_name = random_name('ASTORE')
conn.astore.upload(rstore=table_name, store=store_)
def update_attr(conn, model_name, attr_value, attr_set, attr_key, attr_type):
'''
Update individual extended model attributes
Key/value pair required to specify extended attributes. Provide
correct syntax for calling attribute action.
Parameters
----------
conn : CAS
The CAS connection object
model_name : string
Specifies the name of the deep learning model
attr_value : list of bytes, int, int64, double, or char
Numeric/character representation of attribute
attr_set : string
Name of attribute set to update
attr_key : string
Key name of attribute
attr_type : string
One of double, int64, int, char, or binary
'''
if attr_type.lower() in ['double', 'int64', 'int', 'char', 'binary']:
if attr_type.lower() == 'char':
attr_helper(conn, model_name, attr_set, attr_key, attr_value)
else:
if len(attr_value) > 1:
# create binary blob using SWAT
attr_blob = sw.blob(attr_value)
# write attribute
attr_helper(conn, model_name, attr_set, attr_key, attr_blob)
else:
attr_helper(conn, model_name, attr_set, attr_key, attr_value[0])
else:
raise TypeError('Extended table attributes must be one of :\n'
'1. character string;\n'
'2. double precision value/list,\n'
'3. int64 value/list,\n'
'4. int value/list,\n'
'5. binary blob.')
def upload():
if request.method == 'POST':
# check if the post request has the file part
if 'file' not in request.files:
print('No file part')
return redirect(url_for('index'))
file = request.files['file']
if file.filename == '':
print('No selected file')
return redirect(url_for('index'))
if file:
filename = secure_filename(file.filename)
filepath = os.path.join(UPLOAD_FOLDER, filename)
file.save(filepath)
# CAS Image Processing - load images
s = swat.CAS('localhost', 5570, authinfo=AUTHINFO)
s.loadactionset('image')
s.loadimages(filepath, casout={'name': 'img'})
# Load astore - for now astore exists in / and caslib is session scoped
# TODO - user upload astore
s.loadactionset('astore')
with open('/' + ASTORE + '.astore', 'rb') as f:
store_ = swat.blob(f.read())
s.astore.upload(rstore=ASTORE, store=store_)
# s.score(table={'name':'img'}, out={'name':'score'}, rstore={'name':ASTORE, 'caslib':ASTORE_LIB})
s.score(table={'name': 'img'},
out={'name': 'score'},
rstore={'name': ASTORE})
scores = s.fetch(
table={'name': 'score'})['Fetch'].loc[0, :].to_dict()
label = scores.pop('I__label_')
s.droptable(ASTORE)
s.endSession()
return jsonify({
'imgUrl':
url_for('uploaded_file', filename=filename),
'label':
label,
'scores':
scores
})
return redirect(url_for('index'))
def upload_astore(conn, path, table_name=None):
'''
Load the local astore file to server
Parameters
----------
conn : CAS
The CAS connection object
path : string
Specifies the client-side path of the astore file
table_name : string, or casout options
Specifies the name of the cas table on server to put the astore object
'''
conn.loadactionset('astore')
with open(path, 'br') as f:
astore_byte = f.read()
store_ = sw.blob(astore_byte)
if table_name is None:
table_name = random_name('ASTORE')
conn.astore.upload(rstore=table_name, store=store_)
def test_quartet_fit(self):
if self.server_dir is None:
unittest.TestCase.skipTest(
self,
"DLPY_DATA_DIR_SERVER is not set in the environment variables")
# test using one data table
resnet18_model = ResNet18_Caffe(self.s,
width=224,
height=224,
random_crop='RESIZETHENCROP',
random_flip='HV',
random_mutation='random')
embedding_layer = Dense(n=4)
model1 = EmbeddingModel.build_embedding_model(
resnet18_model,
model_table='test1',
embedding_model_type='quartet',
margin=-3.0,
embedding_layer=embedding_layer)
res1 = model1.print_summary()
print(res1)
img_path = self.server_dir + 'DogBreed_small'
my_images = ImageEmbeddingTable.load_files(
self.s,
path=img_path,
n_samples=64,
embedding_model_type='quartet')
solver = AdamSolver(lr_scheduler=StepLR(learning_rate=0.0001,
step_size=20),
clip_grad_max=100,
clip_grad_min=-100)
optimizer = Optimizer(algorithm=solver,
mini_batch_size=8,
log_level=3,
max_epochs=5,
reg_l2=0.0001)
gpu = Gpu(devices=1)
train_res = model1.fit_embedding_model(data=my_images,
optimizer=optimizer,
n_threads=1,
gpu=gpu,
seed=1234,
record_seed=23435)
print(train_res)
score_res = model1.predict(data=my_images,
gpu=gpu,
random_crop='RESIZETHENCROP')
print(score_res)
# test deploy as astore
self.s.loadactionset('astore')
my_images_test = ImageEmbeddingTable.load_files(
self.s,
path=img_path,
n_samples=5,
embedding_model_type='quartet',
resize_width=224,
resize_height=224)
# case 1: deploy the full model as astore
model1.deploy_embedding_model(output_format='astore',
model_type='full',
path=self.local_dir)
full_astore = os.path.join(self.local_dir,
model1.model_name + '.astore')
with open(full_astore, mode='rb') as file:
file_content = file.read()
store_ = swat.blob(file_content)
self.s.astore.upload(rstore=dict(name='test1_full', replace=True),
store=store_)
# run with one gpu
self.s.score(
rstore=dict(name='test1_full'),
table=my_images_test,
nthreads=1,
# _debug=dict(ranks=0),
copyvars=['_path_', '_path_1', '_path_2', '_path_3'],
options=[
dict(name='usegpu', value='1'),
dict(name='NDEVICES', value='1'),
dict(name='DEVICE0', value='0')
],
out=dict(name='astore_score1_full_gpu', replace=True))
res = self.s.fetch(table='astore_score1_full_gpu')
print(res)
# remove the astore file
os.remove(full_astore)
# case 2: deploy the branch model as astore
model1.deploy_embedding_model(output_format='astore',
model_type='branch',
path=self.local_dir)
br_astore = os.path.join(self.local_dir,
model1.model_name + '_branch.astore')
with open(br_astore, mode='rb') as file:
file_content = file.read()
store_ = swat.blob(file_content)
self.s.astore.upload(rstore=dict(name='test1_br', replace=True),
store=store_)
# run with one gpu
self.s.score(
rstore=dict(name='test1_br'),
table=my_images_test,
nthreads=1,
# _debug=dict(ranks=0),
copyvars=['_path_'],
options=[
dict(name='usegpu', value='1'),
dict(name='NDEVICES', value='1'),
dict(name='DEVICE0', value='0')
],
out=dict(name='astore_score1_br_gpu', replace=True))
res = self.s.fetch(table='astore_score1_br_gpu')
print(res)
os.remove(br_astore)
