def model_update_add(args, method, lr_lists):
model_name = args.model
git_ignore_folder = args.repo
dataset_name = args.dataset
num_epochs = args.epochs
batch_size = args.bz
is_GPU = args.GPU
# args.ratio
regularization_coeff = args.wd
if not is_GPU:
device = torch.device("cpu")
else:
GPU_ID = int(args.GID)
device = torch.device("cuda:"+str(GPU_ID) if torch.cuda.is_available() else "cpu")
model_class = getattr(sys.modules[__name__], model_name)
data_preparer = Data_preparer()
dataset_train = torch.load(git_ignore_folder + "dataset_train")
dataset_test = torch.load(git_ignore_folder + "dataset_test")
X_to_add = torch.load(git_ignore_folder + 'X_to_add')
Y_to_add = torch.load(git_ignore_folder + 'Y_to_add')
learning_rate_all_epochs = torch.load(git_ignore_folder + 'learning_rate_all_epochs')
# generate_random_id_add(git_ignore_folder, dataset_train, num_epochs)
random_ids_all_epochs = torch.load(git_ignore_folder + 'random_ids_multi_epochs')
mini_batch_num = int((len(dataset_train) - 1)/batch_size) + 1
para_list_all_epochs = torch.load(git_ignore_folder + 'para_list_all_epochs')
gradient_list_all_epochs = torch.load(git_ignore_folder + 'gradient_list_all_epochs')
# data_train_loader = torch.load(git_ignore_folder + "data_train_loader")
#
# data_test_loader = torch.load(git_ignore_folder + "data_test_loader")
dim = [len(dataset_train), len(dataset_train[0][0])]
origin_train_data_size = len(dataset_train)
num_class = get_data_class_num_by_name(data_preparer, dataset_name)
hyper_para_function=getattr(Data_preparer, "get_hyperparameters_" + dataset_name)
model = model_class(dim[1], num_class)
init_para_list = list(torch.load(git_ignore_folder + 'init_para'))
init_model(model,init_para_list)
print('data dimension::',dim)
if is_GPU:
model.to(device)
# init_model_params = list(model.parameters())
criterion, optimizer = hyper_para_function(data_preparer, model.parameters(), lr_lists[0], regularization_coeff)
# hyper_params = [criterion, optimizer]
if method == baseline_method:
added_random_ids_multi_epochs = get_sampling_each_iteration0(random_ids_all_epochs, X_to_add.shape[0], mini_batch_num, len(dataset_train))
print("delta data size::", X_to_add.shape[0])
torch.save(added_random_ids_multi_epochs, git_ignore_folder + 'added_random_ids_multi_epochs')
# added_random_ids_multi_epochs = torch.load(git_ignore_folder + 'added_random_ids_multi_epochs')
dataset_train.data = torch.cat([dataset_train.data, X_to_add], 0)
dataset_train.labels = torch.cat([dataset_train.labels, Y_to_add], 0)
t1 = time.time()
updated_model, exp_para_list, exp_grad_list = model_update_standard_lib_add(num_epochs, dataset_train, dim, model, random_ids_all_epochs, batch_size, learning_rate_all_epochs, added_random_ids_multi_epochs, criterion, optimizer, is_GPU, device, regularization_coeff)
t2 = time.time()
process = psutil.Process(os.getpid())
print('memory usage::', process.memory_info().rss)
print('time_baseline::', t2 - t1)
origin_model = torch.load(git_ignore_folder + 'origin_model')
compute_model_para_diff(list(origin_model.parameters()), list(updated_model.parameters()))
torch.save(updated_model, git_ignore_folder + 'model_base_line')
torch.save(exp_para_list, git_ignore_folder + 'exp_para_list')
torch.save(exp_grad_list, git_ignore_folder + 'exp_grad_list')
else:
if method == deltagrad_method:
added_random_ids_multi_epochs = torch.load(git_ignore_folder + 'added_random_ids_multi_epochs')
dataset_train.data = torch.cat([dataset_train.data, X_to_add], 0)
dataset_train.labels = torch.cat([dataset_train.labels, Y_to_add], 0)
exp_para_list = torch.load(git_ignore_folder + 'exp_para_list')
exp_grad_list = torch.load(git_ignore_folder + 'exp_grad_list')
period = args.period
init_epochs = args.init
m = args.m
cached_size = args.cached_size
grad_list_all_epochs_tensor, para_list_all_epoch_tensor, grad_list_GPU_tensor, para_list_GPU_tensor = cache_grad_para_history(git_ignore_folder, cached_size, is_GPU, device)
# model_update_provenance_test3(period, 1, init_epochs, dataset_train, model, grad_list_all_epoch_tensor, para_list_all_epoch_tensor, grad_list_GPU_tensor, para_list_GPU_tensor, cached_size, max_epoch, 2, learning_rate_all_epochs, random_ids_multi_epochss, sorted_ids_multi_epochss, batch_size, dim, added_random_ids_multi_epochs, X_to_add, Y_to_add, criterion, optimizer, lr_scheduler, regularization_coeff, is_GPU, device)
t1 = time.time()
updated_model = model_update_delta_grad_add(exp_para_list, exp_grad_list, period, 1, init_epochs, dataset_train, model, grad_list_all_epochs_tensor, para_list_all_epoch_tensor, grad_list_GPU_tensor, para_list_GPU_tensor, cached_size, m, learning_rate_all_epochs, random_ids_all_epochs, batch_size, dim, added_random_ids_multi_epochs, criterion, optimizer, regularization_coeff, is_GPU, device)
t2 = time.time()
process = psutil.Process(os.getpid())
print('memory usage::', process.memory_info().rss)
print('time_deltagrad::', t2 - t1)
model_base_line = torch.load(git_ignore_folder + 'model_base_line')
compute_model_para_diff(list(model_base_line.parameters()), list(updated_model.parameters()))
torch.save(updated_model, git_ignore_folder + 'model_deltagrad')
is_GPU = bool(int(sys_argv[9]))
if not is_GPU:
device = torch.device("cpu")
else:
GPU_ID = int(sys_argv[10])
device = torch.device("cuda:"+str(GPU_ID) if torch.cuda.is_available() else "cpu")
print(device)
model_class = getattr(sys.modules[__name__], model_name)
data_preparer = Data_preparer()
dataset_train = torch.load(git_ignore_folder + "dataset_train")
dataset_test = torch.load(git_ignore_folder + "dataset_test")
data_train_loader = torch.load(git_ignore_folder + "data_train_loader")
data_test_loader = torch.load(git_ignore_folder + "data_test_loader")
# dataset_train, dataset_test, data_train_loader, data_test_loader = get_train_test_data_loader_by_name_lr(data_preparer, model_class, dataset_name, batch_size)
dim = [len(dataset_train), len(dataset_train[0][0])]
num_class = get_data_class_num_by_name(data_preparer, dataset_name)
def main_add(args, lr_lists):
model_name = args.model
git_ignore_folder = args.repo
dataset_name = args.dataset
num_epochs = args.epochs
batch_size = args.bz
is_GPU = args.GPU
# args.ratio
regularization_coeff = args.wd
if not is_GPU:
device = torch.device("cpu")
else:
GPU_ID = int(args.GID)
device = torch.device("cuda:"+str(GPU_ID) if torch.cuda.is_available() else "cpu")
model_class = getattr(sys.modules[__name__], model_name)
data_preparer = Data_preparer()
dataset_train = torch.load(git_ignore_folder + "dataset_train")
dataset_test = torch.load(git_ignore_folder + "dataset_test")
generate_random_id_add(git_ignore_folder, dataset_train, num_epochs)
random_ids_all_epochs = torch.load(git_ignore_folder + 'random_ids_multi_epochs')
# data_train_loader = torch.load(git_ignore_folder + "data_train_loader")
#
# data_test_loader = torch.load(git_ignore_folder + "data_test_loader")
dim = [len(dataset_train), len(dataset_train[0][0])]
num_class = get_data_class_num_by_name(data_preparer, dataset_name)
hyper_para_function=getattr(Data_preparer, "get_hyperparameters_" + dataset_name)
if model_name == 'Logistic_regression':
model = model_class(dim[1], num_class)
else:
model = model_class()
# model = model_class(dim[1], num_class)
print('data dimension::',dim)
if is_GPU:
model.to(device)
init_model_params = list(model.parameters())
criterion, optimizer = hyper_para_function(data_preparer, model.parameters(), lr_lists[0], regularization_coeff)
hyper_params = [criterion, optimizer]
# lrs = ast.literal_eval(input)#map(float, input.strip('[]').split(','))
# [2.0, 3.0, 4.0, 5.0]
# model, gradient_list_all_epochs, para_list_all_epochs, learning_rate_all_epochs, all_ids_list_all_epochs = model_training_skipnet(num_epochs, model, data_train_loader, data_test_loader, len(dataset_train), len(dataset_test), optimizer, criterion, lr_scheduler, batch_size, is_GPU, device, lrs)
# net, gradient_list_all_epochs, para_list_all_epochs, learning_rate_all_epochs, X_theta_prod_seq, X_theta_prod_softmax_seq, random_ids_multi_epochs
t1 = time.time()
# model, gradient_list_all_epochs, para_list_all_epochs, learning_rate_all_epochs, X_theta_prod_seq, X_theta_prod_softmax_seq, random_ids_multi_epochs = model_training_lr(num_epochs, model, dataset_train, data_test_loader, len(dataset_train), len(dataset_test), optimizer, criterion, lr_scheduler, batch_size, is_GPU, device, lrs)
model, gradient_list_all_epochs, para_list_all_epochs, learning_rate_all_epochs = model_training_lr_test(random_ids_all_epochs, num_epochs, model, dataset_train, len(dataset_train), optimizer, criterion, batch_size, is_GPU, device, lr_lists)
t2 = time.time()
t3 = time.time()
# capture_provenance(git_ignore_folder, data_train_loader, len(dataset_train), dim, num_epochs, num_class, batch_size, int((dim[0] - 1)/batch_size) + 1, torch.stack(random_ids_multi_epochs), X_theta_prod_softmax_seq, X_theta_prod_seq)
# data_train_loader.batch_sampler.reset_ids()
# x_sum_by_class_by_batch = compute_x_sum_by_class_by_batch(data_train_loader, len(dataset_train), batch_size, num_class, random_ids_multi_epochs)
# data_train_loader.batch_sampler.reset_ids()
t4 = time.time()
print("training time full::", t2 - t1)
print("provenance prepare time::", t4 - t3)
torch.save(dataset_train, git_ignore_folder + "dataset_train")
torch.save(gradient_list_all_epochs, git_ignore_folder + 'gradient_list_all_epochs')
torch.save(para_list_all_epochs, git_ignore_folder + 'para_list_all_epochs')
torch.save(learning_rate_all_epochs, git_ignore_folder + 'learning_rate_all_epochs')
torch.save(num_epochs, git_ignore_folder+'epoch')
torch.save(hyper_params, git_ignore_folder + 'hyper_params')
save_random_id_orders(git_ignore_folder, random_ids_all_epochs)
torch.save(para_list_all_epochs[0], git_ignore_folder + 'init_para')
torch.save(model, git_ignore_folder + 'origin_model')
torch.save(model_class, git_ignore_folder + 'model_class')
torch.save(regularization_coeff, git_ignore_folder + 'beta')
torch.save(dataset_name, git_ignore_folder + 'dataset_name')
torch.save(batch_size, git_ignore_folder + 'batch_size')
torch.save(device, git_ignore_folder + 'device')
torch.save(is_GPU, git_ignore_folder + 'is_GPU')
test(model, dataset_test, batch_size, criterion, len(dataset_test), is_GPU, device)