def load_uci_update(role, ip, server, port, mpc_model_dir, mpc_model_filename, updated_model_dir):
"""
Load, update and save uci MPC model.
"""
place = fluid.CPUPlace()
exe = fluid.Executor(place)
# Step 1. initialize MPC environment and load MPC model into default_main_program to update.
pfl_mpc.init("aby3", role, ip, server, port)
aby3.load_mpc_model(exe=exe,
mpc_model_dir=mpc_model_dir,
mpc_model_filename=mpc_model_filename)
# Step 2. MPC update
epoch_num = network.MPC_UPDATE_EPOCH
batch_size = network.BATCH_SIZE
mpc_data_dir = "../mpc_data/"
feature_file = mpc_data_dir + "house_feature"
feature_shape = (13,)
label_file = mpc_data_dir + "house_label"
label_shape = (1,)
loss_file = "./tmp/uci_mpc_loss.part{}".format(role)
if os.path.exists(loss_file):
os.remove(loss_file)
updated_model_name = 'mpc_updated_model'
feature_name = 'x'
label_name = 'y'
# fetch loss if needed
loss = fluid.default_main_program().global_block().var('mean_0.tmp_0')
loader = process_data.get_mpc_dataloader(feature_file, label_file, feature_shape, label_shape,
feature_name, label_name, role, batch_size)
start_time = time.time()
for epoch_id in range(epoch_num):
step = 0
for sample in loader():
mpc_loss = exe.run(feed=sample, fetch_list=[loss.name])
if step % 50 == 0:
print('Epoch={}, Step={}, Loss={}'.format(epoch_id, step, mpc_loss))
with open(loss_file, 'ab') as f:
f.write(np.array(mpc_loss).tostring())
step += 1
end_time = time.time()
print('Mpc Updating of Epoch={} Batch_size={}, cost time in seconds:{}'
.format(epoch_num, batch_size, (end_time - start_time)))
# Step 3. save updated MPC model as a trainable model.
aby3.save_trainable_model(exe=exe,
model_dir=updated_model_dir,
model_filename=updated_model_name)
print('Successfully save mpc updated model into:{}'.format(updated_model_dir))
def load_mpc_model_and_predict(role, ip, server, port, mpc_model_dir,
mpc_model_filename):
"""
Predict based on MPC inference model, save prediction results into files.
"""
place = fluid.CPUPlace()
exe = fluid.Executor(place)
# Step 1. initialize MPC environment and load MPC model to predict
pfl_mpc.init("aby3", role, ip, server, port)
infer_prog, feed_names, fetch_targets = aby3.load_mpc_model(
exe=exe,
mpc_model_dir=mpc_model_dir,
mpc_model_filename=mpc_model_filename,
inference=True)
# Step 2. MPC predict
batch_size = network.BATCH_SIZE
feature_file = "/tmp/house_feature"
feature_shape = (13, )
pred_file = "./tmp/uci_prediction.part{}".format(role)
loader = process_data.get_mpc_test_dataloader(feature_file, feature_shape,
role, batch_size)
start_time = time.time()
for sample in loader():
prediction = exe.run(program=infer_prog,
feed={feed_names[0]: np.array(sample)},
fetch_list=fetch_targets)
# Step 3. save prediction results
with open(pred_file, 'ab') as f:
f.write(np.array(prediction).tostring())
break
end_time = time.time()
print('Mpc Predict with samples of {}, cost time in seconds:{}'.format(
batch_size, (end_time - start_time)))
def infer(args):
"""
infer
"""
logger.info('Start inferring...')
begin = time.time()
place = fluid.CUDAPlace(0) if args.use_gpu else fluid.CPUPlace()
exe = fluid.Executor(place)
cur_model_path = os.path.join(args.model_dir, 'mpc_model', 'epoch_' + str(args.test_epoch),
'checkpoint', 'party_{}'.format(args.role))
with fluid.scope_guard(fluid.Scope()):
pfl_mpc.init('aby3', args.role, 'localhost', args.server, args.port)
infer_program, feed_target_names, fetch_vars = aby3.load_mpc_model(exe=exe,
mpc_model_dir=cur_model_path,
mpc_model_filename='__model__',
inference=True)
mpc_data_dir = args.mpc_data_dir
user_vec_filepath = mpc_data_dir + 'user_vec'
user_vec_part_filepath = user_vec_filepath + '.part{}'.format(args.role)
sample_batch = args.batch_size
watch_vecs = []
search_vecs = []
other_feats = []
watch_vec_reader = read_share(file=mpc_data_dir + 'watch_vec', shape=(sample_batch, args.watch_vec_size))
for vec in watch_vec_reader():
watch_vecs.append(vec)
search_vec_reader = read_share(file=mpc_data_dir + 'search_vec', shape=(sample_batch, args.search_vec_size))
for vec in search_vec_reader():
search_vecs.append(vec)
other_feat_reader = read_share(file=mpc_data_dir + 'other_feat', shape=(sample_batch, args.other_feat_size))
for vec in other_feat_reader():
other_feats.append(vec)
if os.path.exists(user_vec_part_filepath):
os.system('rm -rf ' + user_vec_part_filepath)
for i in range(args.batch_num):
l3 = exe.run(infer_program,
feed={
'watch_vec': watch_vecs[i],
'search_vec': search_vecs[i],
'other_feat': other_feats[i],
},
return_numpy=True,
fetch_list=fetch_vars)
with open(user_vec_part_filepath, 'a+') as f:
f.write(np.array(l3[0]).tostring())
end = time.time()
logger.info('MPC inferring, cost_time: {:.5f}s'.format(end - begin))
logger.info('End inferring.')
def infer(test_loader, role, exe, BATCH_SIZE, mpc_model_dir,
mpc_model_filename):
"""
load mpc model and infer
"""
# Load mpc model
logger.info('Load model from {}'.format(mpc_model_dir))
infer_program, feed_targets, fetch_targets = aby3.load_mpc_model(
exe=exe,
mpc_model_dir=mpc_model_dir,
mpc_model_filename=mpc_model_filename,
inference=True)
# Infer
logger.info('******************************************')
logger.info('Start Inferring...')
mpc_infer_data_dir = "./mpc_infer_data/"
if not os.path.exists(mpc_infer_data_dir):
try:
os.mkdir(mpc_infer_data_dir)
except OSError as e:
if e.errno != errno.EEXIST:
raise
prediction_file = mpc_infer_data_dir + "prediction.part{}".format(role)
if os.path.exists(prediction_file):
os.remove(prediction_file)
start_time = time.time()
for sample in test_loader():
prediction = exe.run(program=infer_program,
feed=sample,
fetch_list=fetch_targets)
with open(prediction_file, 'ab') as f:
f.write(np.array(prediction).tostring())
end_time = time.time()
logger.info('End Inferring...cost time: {}'.format(end_time - start_time))
logger.info('Start Evaluate Accuracy...')
cypher_file = mpc_infer_data_dir + "prediction"
decrypt_file = mpc_infer_data_dir + 'label_mpc'
time.sleep(0.1)
if role == 0:
if os.path.exists(decrypt_file):
os.remove(decrypt_file)
process_data.decrypt_data_to_file(cypher_file, (BATCH_SIZE, ),
decrypt_file)
evaluate.evaluate_accuracy('./mpc_infer_data/label_criteo',
decrypt_file)
evaluate.evaluate_auc('./mpc_infer_data/label_criteo', decrypt_file)
end_time = time.time()
logger.info('End Evaluate Accuracy...cost time: {}'.format(end_time -
start_time))
logger.info('******************************************')