def ewc_process_without_split(train_loader, test_loader, labels, online=False, result_file='./ewc_without_split.txt'):
gpu = torch.device('cuda:0')
model = model_retrieval().cuda(gpu)
optimizer = optim.SGD(params=model.parameters(), lr=args.first_lr)
ewcs = []
for task in range(args.num_task):
print('Training Task {}... Labels: {}'.format(task, labels[task]))
if task == 0:
for iteration in range(args.iterations):
loss = normal_train(model, labels[task], optimizer, train_loader[task], gpu)
print('Iteration: {}\tLoss:{}'.format(iteration, loss))
acc = test_model(model, labels[task], test_loader[task], gpu)
print('Test Task: {}\tAccuracy: {}'.format(task, acc))
with open(result_file, 'a') as f:
f.write('{}\t{}\t{}\t{}\t{}\t{}\n'.format(task, iteration, loss, task, task, acc))
else:
for param_group in optimizer.param_groups:
param_group['lr'] = args.lr
for iteration in range(args.iterations):
if online:
loss = ewc_train(model, labels[task], optimizer, train_loader[task], ewcs[-1:], args.lam, gpu)
else:
loss = ewc_train(model, labels[task], optimizer, train_loader[task], ewcs, args.lam, gpu)
print('Iteration: {}\tLoss:{}'.format(iteration, loss))
for sub_task in range(task + 1):
acc = test_model(model, labels[sub_task], test_loader[sub_task], gpu)
print('Test Task: {}\tAccuracy: {}'.format(sub_task, acc))
with open(result_file, 'a') as f:
# Current server parameter (task) + device parameter (task) --> training a given task
f.write('{}\t{}\t{}\t{}\t{}\n'.format(task, iteration, loss, sub_task, acc))
ewcs.append(EWC(model, train_loader[task], gpu))
def train_model(self, fname=None):
time.sleep(1)
try:
self.load(fname)
print("File load successful.")
except Exception:
print("File load failed.")
image_dataset, dataset_size = utils.create_dataset(
batch_size=batch_size)
dataset_size = dataset_size - 1
print("Dataset size is", dataset_size)
print("Total number of images is", dataset_size * batch_size)
record_batch = utils.record_steps(int(dataset_size / 2))
start_time = time.time()
print("Beginning training at", start_time)
for i in range(num_epochs):
print("Starting epoch {}/{}".format(i, num_epochs))
start = time.time()
batch_on = 0
for source in zip(image_dataset.take(int(dataset_size / 2))):
loss_identity, kl_loss = train_step(self, source, optimizer)
if batch_on % record_batch == 0:
print("Beginning batch #" + str(batch_on), 'out of',
int(dataset_size / 2), 'of size', batch_size)
self.loss_identity += [loss_identity]
self.kl_loss += [kl_loss]
batch_on += 1
duration = time.time() - start
print(int(duration / 60), "minutes &", int(duration % 60),
"seconds, for epoch", i)
if i % record_epochs == 0:
self.loss['Identity'] = self.loss_identity
self.loss['KL'] = self.kl_loss
utils.test_model(self,
source,
num=i,
test=True,
name=model_name,
details='identity')
for style in zip(image_dataset.take(1)):
utils.test_model(self,
source,
style,
num=i,
test=True,
name=model_name,
details='transfer')
break
#act = keract.get_activations(self, source)
#keract.display_activations(act)
print('\n')
image_dataset, _ = utils.create_dataset(batch_size=batch_size)
self.save(fname)
time.sleep(.5)
print('Training completed in', int((time.time() - start_time) / 60),
"minutes &", int(duration % 60), "seconds")
def _train(model, optimizer, scheduler, checkpointer, epochs, train_loader,
test_loader, stat_tracker, log_dir, device):
'''
Training loop to train classifiers on top of an encoder with fixed weights.
-- e.g., use this for eval or running on new data
'''
# If mixed precision is on, will add the necessary hooks into the model and
# optimizer for half precision conversions
model, optimizer = mixed_precision.initialize(model, optimizer)
# ...
time_start = time.time()
total_updates = 0
next_epoch, total_updates = checkpointer.get_current_position(
classifier=True)
for epoch in range(next_epoch, epochs):
epoch_updates = 0
epoch_stats = AverageMeterSet()
for _, ((images1, images2), labels) in enumerate(train_loader):
# get data and info about this minibatch
images1 = images1.to(device)
images2 = images2.to(device)
labels = labels.to(device)
# run forward pass through model and collect activations
res_dict = model(x1=images1, x2=images2, class_only=True)
lgt_glb_mlp, lgt_glb_lin = res_dict['class']
# compute total loss for optimization
loss = (loss_xent(lgt_glb_mlp, labels) +
loss_xent(lgt_glb_lin, labels))
# do optimizer step for encoder
optimizer.zero_grad()
mixed_precision.backward(
loss, optimizer) # special mixed precision stuff
optimizer.step()
# record loss and accuracy on minibatch
epoch_stats.update('loss', loss.item(), n=1)
update_train_accuracies(epoch_stats, labels, lgt_glb_mlp,
lgt_glb_lin)
# shortcut diagnostics to deal with long epochs
total_updates += 1
epoch_updates += 1
if (total_updates % 100) == 0:
time_stop = time.time()
spu = (time_stop - time_start) / 100.
print(
'Epoch {0:d}, {1:d} updates -- {2:.4f} sec/update'.format(
epoch, epoch_updates, spu))
time_start = time.time()
# step learning rate scheduler
scheduler.step(epoch)
# record diagnostics
test_model(model, test_loader, device, epoch_stats, max_evals=500000)
epoch_str = epoch_stats.pretty_string(ignore=model.tasks)
diag_str = '{0:d}: {1:s}'.format(epoch, epoch_str)
print(diag_str)
sys.stdout.flush()
stat_tracker.record_stats(epoch_stats.averages(epoch, prefix='eval/'))
checkpointer.update(epoch + 1, total_updates, classifier=True)
def train(_config, resume: bool = False, test: bool = False):
print(json.dumps(config, indent=4))
device = torch.device(_config['device'])
os.environ["CUDA_VISIBLE_DEVICES"] = str(device.index)
device = torch.device(0)
dataset = _config['data']['dataset']
model_name = _config['model']['name']
optimizer_name = _config['optimizer']['name']
scheduler_name = _config['scheduler']['name']
loss = utils.get_loss(_config['loss']['name'])
loss.to(device)
model = create_model(dataset, _config['model'][model_name],
_config['model']['stadaptor'], device)
optimizer = utils.get_optimizer(optimizer_name, model.parameters(),
**_config['optimizer'][optimizer_name])
scheduler = None
if scheduler_name is not None:
scheduler = utils.get_scheduler(scheduler_name, optimizer,
**_config['scheduler'][scheduler_name])
save_folder = os.path.join('saves', dataset, _config['name'])
if not resume and not test:
shutil.rmtree(save_folder, ignore_errors=True)
os.makedirs(save_folder)
with open(os.path.join(save_folder, 'config.yaml'), 'w+') as _f:
yaml.safe_dump(_config, _f)
datasets = utils.get_datasets(dataset, _config['data']['input_dim'],
_config['data']['output_dim'])
scaler = utils.ZScoreScaler(datasets['train'].mean, datasets['train'].std)
trainer = utils.OursTrainer(model, loss, scaler, device, optimizer,
**_config['trainer'])
if not test:
utils.train_model(datasets=datasets,
batch_size=_config['data']['batch-size'],
folder=save_folder,
trainer=trainer,
scheduler=scheduler,
epochs=config['epochs'],
early_stop_steps=config['early_stop_steps'])
utils.test_model(datasets=datasets,
batch_size=_config['data']['batch-size'],
trainer=trainer,
folder=save_folder)
def train_model(self, fname=None):
time.sleep(1)
try:
self.load(fname)
print("File load successful.")
except Exception:
print("File load failed.")
model_val = AE_A(training=False)
image_dataset, dataset_size = utils.create_dataset(batch_size=batch_size)
dataset_size = int((dataset_size - 1)/2)
print("Dataset size is", dataset_size)
print("Total number of images is", dataset_size * batch_size)
record_batch = utils.record_steps(dataset_size)
start_time = time.time()
for i in range(num_epochs):
print(f'Starting epoch {i}/{num_epochs}')
start = time.time()
batch_on = 0
for source in zip(image_dataset.take(dataset_size)):
try:
loss_identity, kl_loss = self.train_step(source, optimizer)
if batch_on % record_batch == 0:
print(f'Beginning batch #{batch_on} out of {dataset_size} of size {batch_size}')
self.loss_identity += [loss_identity]
self.kl_loss += [kl_loss]
except Exception:
print(f'Batch #{batch_on} failed. Continuing with next batch.')
batch_on += 1
self.save(fname)
time.sleep(0.5)
duration = time.time() - start
utils.print_time_remaining(i, num_epochs, duration)
if i % display_mod == 0:
model_val.load(model_name, compile=False)
model_val.loss['Identity'] = self.loss_identity
model_val.loss['KL'] = self.kl_loss
utils.test_model(model_val, source, num=i, test=True, name=model_name, details='identity')
for style in zip(image_dataset.take(1)):
utils.test_model(model_val, source, style, num=i, test=True, name=model_name, details='merge')
break
#act = keract.get_activations(self, source)
#keract.display_activations(act)
print('\n')
image_dataset, _ = utils.create_dataset(batch_size=batch_size)
print('Training completed in', int((time.time() - start_time) / 60), "minutes &", int(duration % 60), "seconds")
def build_model(data_x,
data_y,
split_mode='test',
feature_method='f_regression',
feature_num=100,
pkl_dir='pkl'):
# select features
# feature_num: integer that >=0
# method: ['f_regression', 'mutual_info_regression', 'pca']
data_x, min_max = ut.select_feature(data_x,
data_y,
method=feature_method,
feature_num=feature_num)
#if min_max is not None:
# min_max.to_csv(os.path.join(pkl_dir, 'min_max.csv'), index=False, index_label=False, encoding='gb18030')
# start building model
np_x = np.nan_to_num(data_x.values)
np_y = np.nan_to_num(data_y.values)
print('train_set.shape=%s, test_set.shape=%s' % (np_x.shape, np_y.shape))
res = {}
if split_mode == 'test':
x_train, x_val, y_train, y_val = train_test_split(data_x,
data_y,
test_size=0.2,
shuffle=True)
model = LinearRegression()
res['lr'] = ut.test_model(model, x_train, x_val, y_train, y_val)
ut.save_model(model, data_x.columns, pkl_dir)
model = DecisionTreeRegressor()
res['dt'] = ut.test_model(model, x_train, x_val, y_train, y_val)
ut.save_model(model, data_x.columns, pkl_dir, depth=5)
model = RandomForestRegressor(
) #criterion='mse', n_estimators=500, min_samples_leaf=50)
res['rf'] = ut.test_model(model, x_train, x_val, y_train, y_val)
ut.save_model(model, data_x.columns, pkl_dir, depth=5)
model = GradientBoostingRegressor()
res['gbdt'] = ut.test_model(model, x_train, x_val, y_train, y_val)
ut.save_model(model, data_x.columns, pkl_dir)
elif split_mode == 'cv':
model = LinearRegression()
res['lr'] = ut.cv_model(model, np_x, np_y)
model = DecisionTreeRegressor()
res['dt'] = ut.cv_model(model, np_x, np_y)
model = RandomForestRegressor()
res['rf'] = ut.cv_model(model, np_x, np_y)
model = GradientBoostingRegressor()
res['gbdt'] = ut.cv_model(model, np_x, np_y)
else:
print('parameter mode=%s unresolved' % (model))
return res
def train_model(self, fname=None):
time.sleep(1)
try:
self.load(fname)
print("File load successful.")
except Exception:
print("File load failed.")
image_dataset, dataset_size = utils.create_dataset(
batch_size=batch_size)
dataset_size = dataset_size - 1
print("Dataset size is", dataset_size)
print("Total number of images is", dataset_size * batch_size)
record_batch = utils.record_steps(int(dataset_size))
start_time = time.time()
print("Beginning training at", start_time)
for i in range(num_epochs):
print("Starting epoch {}/{}".format(i, num_epochs))
start = time.time()
batch_on = 0
for source in zip(image_dataset.take(int(dataset_size))):
source = utils.get_random_crop(np.array(source), 32, 32)
loss_identity = train_step(self, source, optimizer)
if batch_on % record_batch == 0:
print("Beginning batch #" + str(batch_on), 'out of',
int(dataset_size), 'of size', batch_size)
self.loss_identity += [loss_identity]
batch_on += 1
if i % record_epochs == 0:
self.loss['Identity'] = self.loss_identity
utils.test_model(self,
source,
num=i,
test=True,
name=model_name,
details='identity')
'''for style in zip(image_dataset.take(1)):
style = utils.get_random_crop(style, 32, 32)
utils.test_model(self, source, style, num=i, test=True, name=model_name, details='transfer')
break'''
print('\n')
duration = time.time() - start
utils.print_time_remaining(i, num_epochs, duration)
image_dataset, _ = utils.create_dataset(batch_size=batch_size)
self.save(fname)
time.sleep(.5)
print('Training completed in', int((time.time() - start_time) / 60),
"minutes &", int(duration % 60), "seconds")
def standard_process(train_loader, test_loader, labels, result_file='./standard.txt'):
gpu = torch.device('cuda:0')
new_model = model_retrieval().cuda(gpu)
# print_model(new_model)
models = [copy.deepcopy(new_model) for _ in range(args.num_task)]
# time.sleep(10)
temp_model = copy.deepcopy(new_model)
cut_idx = generate_cut_layer(args.split, temp_model)
optimizers = [optim.SGD(params=models[idx].parameters(), lr=args.lr) for idx in range(args.num_task)]
for task in range(args.num_task):
print('Training Task {}... Labels: {}'.format(task, labels[task]))
model, optimizer = models[task] if task == 0 else models_copy(models[task], models[task-1], cut_idx), optimizers[task]
# print_model(model)
if task == 0:
for param_group in optimizer.param_groups:
param_group['lr'] = args.first_lr
for iteration in range(args.iterations):
loss = normal_train(model, labels[task], optimizer, train_loader[task], gpu)
# print_model(model)
print('Iteration: {}\tLoss:{}'.format(iteration, loss))
for sub_task in range(task + 1):
temp_model = copy.deepcopy(models[sub_task])
temp_model = models_copy(temp_model, model, cut_idx)
for i in range(task + 1):
acc = test_model(temp_model, labels[i], test_loader[i], gpu)
print('Device Task: {}\tTest Task: {}\tAccuracy: {}'.format(sub_task, i, acc))
with open(result_file, 'a') as f:
# Current server parameter (task) + device parameter (task) --> training a given task
f.write('{}\t{}\t{}\t{}\t{}\t{}\n'.format(task, iteration, loss, sub_task, i, acc))
def train_model(self, fname=None):
time.sleep(5)
try:
self.load(fname)
print("File load successful.")
except Exception:
print("File load failed.")
image_dataset, dataset_size = utils.create_dataset()
dataset_size = dataset_size - 1
print("Dataset size is", dataset_size)
print("Total number of images is", dataset_size * batch_size)
start_time = time.time()
print("Beginning training at", start_time)
for i in range(num_epochs):
print("Starting epoch {}/{}".format(i, num_epochs))
start = time.time()
batch_on = 0
for source, style in zip(image_dataset.take(int(dataset_size / 4)),
image_dataset.take(int(dataset_size /
4))):
try:
loss_content, loss_style, loss_identity = self.train_step(
source, style, optimizer)
if batch_on % 10 == 0:
print("Beginning batch #" + str(batch_on), 'out of',
int(dataset_size / 4), 'of size', batch_size)
self.loss_content += [loss_content]
self.loss_style += [loss_style]
self.loss_identity += [loss_identity]
except Exception:
print("Batch #", batch_on,
"failed. Continuing with next batch.")
batch_on += 1
duration = time.time() - start
print(int(duration / 60), "minutes &", int(duration % 60),
"seconds, for epoch", i)
if i % 20 == 0:
utils.test_model(self, source, style, num=i, name=model_name)
print('\n')
image_dataset, _ = utils.create_dataset()
self.save(fname)
time.sleep(1)
print('Training completed in', int((time.time() - start_time) / 60),
"minutes &", int(duration % 60), "seconds")
def our_process(train_loader, test_loader, labels, online=False, result_file='./our_process.txt'):
gpu = torch.device('cuda:0')
new_model = model_retrieval().cuda(gpu)
models = [copy.deepcopy(new_model) for _ in range(args.num_task)]
temp_model = copy.deepcopy(new_model)
cut_idx = generate_cut_layer(args.split, temp_model)
optimizers = [optim.SGD(params=models[idx].parameters(), lr=args.lr) for idx in range(args.num_task)]
ewcs = []
if_freeze = 0
for task in range(args.num_task):
print('Training Task {}... Labels: {}'.format(task, labels[task]))
model, optimizer = models[task] if task == 0 else models_copy(models[task], models[task-1], cut_idx), optimizers[task] #model为tmp变量循环用，model：copy了上一次task中model的param（与cut_idx有关）
if task == 0:
for param_group in optimizer.param_groups:
param_group['lr'] = args.first_lr
for iteration in range(args.iterations):
loss = normal_train(model, labels[task], optimizer, train_loader[task], gpu)
print('Iteration: {}\tLoss:{}'.format(iteration, loss))
acc = test_model(model, labels[task], test_loader[task], gpu)
print('Device Task: {}\tTest Task: {}\tAccuracy: {}'.format(task, task, acc))
with open(result_file, 'a') as f:
f.write('{}\t{}\t{}\t{}\t{}\t{}\n'.format(task, iteration, loss, task, task, acc))
else:
for iteration in range(args.iterations):
if online:
loss = our_train(model, labels[task], optimizer, train_loader[task], ewcs[-1:], args.lam, gpu, cut_idx, if_freeze) #与normal_train相比多了ewcs[]与args.lam
else:
loss = our_train(model, labels[task], optimizer, train_loader[task], ewcs, args.lam, gpu, cut_idx, if_freeze) #与online区别是ewcs[-1:]为ewcs只取最后一个元素构成的列表
#判断loss，loss若小于阈值，令变量if_freeze=1,传入下次our_train
#our_train相比于ewc_train多两个参数：if_freeze和cut_idx
if loss < args.threshold:
if_freeze = 1
else:
if_freeze = 0
print('Iteration: {}\tLoss:{}\tif freeze:{}'.format(iteration, loss, if_freeze))
for sub_task in range(task + 1): #循环不同model
temp_model = copy.deepcopy(models[sub_task])
temp_model = models_copy(temp_model, model, cut_idx) #temp_model 用的是当前model后半部分，models[]前半部分
for i in range(task + 1): #循环不同task
acc = test_model(temp_model, labels[i], test_loader[i], gpu)
print('Device Task: {}\tTest Task: {}\tAccuracy: {}'.format(sub_task, i, acc))
with open(result_file, 'a') as f:
# Current server parameter (task) + device parameter (task) --> training a given task
f.write('{}\t{}\t{}\t{}\t{}\t{}\n'.format(task, iteration, loss, sub_task, i, acc))
ewcs.append(splitEWC(model, train_loader[task], cut_idx, gpu))
def cb(generator, progress, batch_index, result):
iteration = (current_task - 1) * total_iterations + batch_index
progress.set_description(
('<Training Generator> '
'task: {task}/{tasks} | '
'progress: [{trained}/{total}] ({percentage:.0f}%) | '
'loss => '
'g: {g_loss:.4} / '
'w: {w_dist:.4}').format(
task=current_task,
tasks=total_tasks,
trained=batch_size * batch_index,
total=batch_size * total_iterations,
percentage=(100. * batch_index / total_iterations),
g_loss=result['g_loss'],
w_dist=-result['c_loss'],
))
# log the losses of the generator.
if iteration % loss_log_interval == 0:
visual.visualize_scalar(result['g_loss'],
'generator g loss',
iteration,
env=env)
visual.visualize_scalar(-result['c_loss'],
'generator w distance',
iteration,
env=env)
# log the generated images of the generator (to visdom).
if iteration % sample_log_interval == 0:
visual.visualize_images(
generator.sample(sample_size).data,
'generated samples ({replay_mode})'.format(
replay_mode=replay_mode),
env=env,
)
# log the sample images of the generator (to file)
if iteration % sample_log_interval == 0 and sample_log:
utils.test_model(generator,
sample_size,
os.path.join(sample_dir, env + '-sample-logs',
str(iteration)),
verbose=False)
def main():
for model in [
'LinearRegression', 'LogisticRegression', 'RandomForest',
'DecisionTree', 'SVM'
]:
display(f'Training model {model}')
def func():
# training model
return train_model(model, x_train, y_train)
trained_model, training_time = timed_func(func)
display(f'time used: {training_time}')
display(f'Testing model')
results, confusion_matrix, accuracy_score = test_model(
trained_model, x_test, y_test,
lambda predictions: [int(i) for i in predictions])
display(f'confusion_matrix: \n{confusion_matrix}')
display(f'accuracy_score: {accuracy_score}\n\n')
def train(continue_from,train_loader,test_loader):
print(continue_from)
#continue_from = "best_transformer_age5.pth"
modeltype = "age"
model = MyModel(modeltype,use_col)
model = model.cuda()
#warm up optimizer
optimizer = TransformerOptimizer(
torch.optim.Adam(model.parameters(), betas=(0.9, 0.98), eps=1e-08),
0.8,
512,
8000)
start_epoch = 0
print("start load")
if os.path.exists(continue_from ):
naive_model = torch.load(continue_from , map_location=lambda storage, loc:storage)
new_state_dict = OrderedDict()
for k, v in naive_model["state"].items():
new_state_dict[k] = v
model.load_state_dict(new_state_dict)
start_epoch = naive_model["epoch"]+1
optimizer.load_state_dict(naive_model["optimizer"])
optimizer.step_num = naive_model['optimizer_step']
logger.info("load model finish")
print("end load")
losses_age = []
losses_gender= []
best_acc_age = 0
best_acc_gender = 0
entroy=nn.CrossEntropyLoss()#mseloss = nn.MSELoss()
model = nn.DataParallel(model)
best_age_acc = 0
best_gender_acc = 0
#acc_age,acc_gender,loss_age,loss_gender = test_model(model,test_loader,entroy)
for epo in range(start_epoch,30):
model.train()
#start = time.time()
for i,(data) in enumerate(train_loader):
raw_arr,click,times,label_age,label_gender,user_id,user_id_emb = data
label_age = label_age.cuda()
label_gender = label_gender.cuda()
click = click.cuda()
times= times.cuda()
user_id_emb = user_id_emb.cuda()
raw_arr= raw_arr.cuda()
embs1,embs2 = model(user_id_emb,raw_arr,click,times)
loss_age = entroy(embs1,label_age-1)
loss_gender = entroy(embs2,label_gender-1)
loss = loss_age+loss_gender
optimizer.zero_grad()
loss.backward()
#torch.nn.utils.clip_grad_norm_(model.parameters(), 400)
optimizer.step()
losses_age.append(loss_age.item())
losses_gender.append(loss_gender.item())
if (i+1)%200==0:
#print((time.time()-start)/200*len(train_loader))
for g in optimizer.optimizer.param_groups:
logger.info("now learn rate : "+str(g['lr']) )
mean_loss_age = sum(losses_age)/len(losses_age)
mean_loss_gender = sum(losses_gender)/len(losses_gender)
logger.info('cpEpoch:{0} \t step:{1}/{2} \t '
'mean loss:{3} \t{4}'.format(
(epo + 1), (i + 1),len(train_loader),mean_loss_age,mean_loss_gender))
acc_age,acc_gender,loss_age,loss_gender = test_model(model,test_loader,entroy)
logger.info("acc:{},{} loss:{}{}".format(acc_age,acc_gender,loss_age,loss_gender))
if acc_age>best_acc_age:
best_acc_age = acc_age
save_model(model,epo,optimizer,model_name = continue_from)
if acc_gender>best_gender_acc:
best_gender_acc = acc_gender
save_model(model,epo,optimizer,model_name = continue_from + "gender.pth")
scholar.generator.set_critic_optimizer(generator_c_optimizer)
# set additional settings for the scholar's generator
scholar.generator.set_lambda(args.lamda)
scholar.generator.set_critic_updates_per_batch(args.cu)
# run the experiment.
if args.train:
train(
scholar, train_datasets, test_datasets,
replay_mode=args.replay_mode,
generator_iterations=args.gen_iter if train_generator else 0,
solver_iterations=args.sol_iter,
importance_of_new_task=args.rnt,
batch_size=args.batch,
test_size=args.test_n,
sample_size=args.s_size,
loss_log_interval=args.loss_log,
eval_log_interval=args.eval_log,
sample_log_interval=args.s_log,
sample_log=args.samples,
sample_dir=args.s_dir,
checkpoint_dir=args.c_dir,
collate_fn=utils.label_squeezing_collate_fn,
cuda=cuda,
valid_proportion=valid_proportion
)
else:
path = os.path.join(args.s_dir, '{}-sample'.format(scholar.name))
utils.load_checkpoint(scholar, args.c_dir)
utils.test_model(scholar.generator, args.s_size, path)
if frozen_predictor:
for param in model.predictor.parameters():
param.requires_grad_(False)
datasets = utils.get_datasets(datasets, 9, 1)
scaler = utils.ZScoreScaler(datasets['train'].mean, datasets['train'].std)
optimizer = optim.Adam([{
'params': model.adaptor.parameters()
}, {
'params': model.predictor.parameters(),
'lr': 1e-5
}],
lr=learning_rate)
loss = utils.get_loss('MaskedMAELoss')
trainer = utils.OursTrainer(model, loss, scaler, device, optimizer,
weight_decay, 2, 5)
utils.train_model(datasets=datasets,
batch_size=64,
folder=saved_folder,
trainer=trainer,
scheduler=None,
epochs=100,
early_stop_steps=10)
utils.test_model(datasets=datasets,
batch_size=64,
trainer=trainer,
folder=saved_folder)
def ewc_process(train_loader,
test_loader,
labels,
class_incremental,
online=False,
result_file='./ewc.txt'):
gpu = torch.device('cuda:0')
new_model = model_retrieval().cuda(gpu)
models, cur_label = [
copy.deepcopy(new_model) for _ in range(args.num_task)
], []
temp_model = copy.deepcopy(new_model)
cut_idx = generate_cut_layer(args.split, temp_model)
optimizers = [
optim.SGD(params=models[idx].parameters(), lr=args.lr)
for idx in range(args.num_task)
]
ewcs = []
for task in range(args.num_task):
print('Training Task {}... Labels: {}'.format(task, labels[task]))
model, optimizer = models[task] if task == 0 else models_copy(
models[task], models[task - 1], cut_idx
), optimizers[
task] #model为tmp变量循环用，model：copy了上一次task中model的param（与cut_idx有关）
if task == 0:
cur_label = cur_label + labels[
task] if class_incremental else labels[task]
for param_group in optimizer.param_groups:
param_group['lr'] = args.first_lr
for iteration in range(args.iterations):
loss = normal_train(model, cur_label, optimizer,
train_loader[task], gpu)
print('Iteration: {}\tLoss:{}'.format(iteration, loss))
acc = test_model(model, cur_label, test_loader[task], gpu)
print('Device Task: {}\tTest Task: {}\tAccuracy: {}'.format(
task, task, acc))
with open(result_file, 'a') as f:
f.write('{}\t{}\t{}\t{}\t{}\t{}\n'.format(
task, iteration, loss, task, task, acc))
if iteration % 20 == 0:
for param_group in optimizer.param_groups:
param_group['lr'] *= 0.95
else:
cur_label = cur_label + labels[
task] if class_incremental else labels[task]
for iteration in range(args.iterations):
if online:
loss = ewc_train(model, cur_label, optimizer,
train_loader[task], ewcs[-1:], args.lam,
gpu) #与normal_train相比多了ewcs[]与args.lam
else:
loss = ewc_train(
model, cur_label, optimizer, train_loader[task], ewcs,
args.lam, gpu) #与online区别是ewcs[-1:]为ewcs只取最后一个元素构成的列表
print('Iteration: {}\tLoss:{}'.format(iteration, loss))
for sub_task in range(task + 1): #循环不同model
temp_model = copy.deepcopy(models[sub_task])
temp_model = models_copy(
temp_model, model,
cut_idx) #temp_model 用的是当前model后半部分，models[]前半部分
for i in range(task + 1): #循环不同task
cur_label = cur_label if class_incremental else labels[
i]
acc = test_model(temp_model, cur_label, test_loader[i],
gpu)
print('Device Task: {}\tTest Task: {}\tAccuracy: {}'.
format(sub_task, i, acc))
with open(result_file, 'a') as f:
# Current server parameter (task) + device parameter (task) --> training a given task
f.write('{}\t{}\t{}\t{}\t{}\t{}\n'.format(
task, iteration, loss, sub_task, i, acc))
if iteration % 20 == 0:
for param_group in optimizer.param_groups:
param_group['lr'] *= 0.95
ewcs.append(splitEWC(model, train_loader[task], cut_idx, gpu))
#rf_classifier = sk_learn_classifiers.train_random_forest(X_train, y_train)
rf_classifier = sk_learn_classifiers.load_model('outputs/random_forest2')
#xgboost_classifier = sk_learn_classifiers.train_xgboost(X_train, y_train)
xgboost_classifier = sk_learn_classifiers.load_model('outputs/xgboost1')
df_X_train_preds_classifiers = get_X_predictions_classifiers(
deep_classifier, rf_classifier, xgboost_classifier, X_train)
df_X_test_preds_classifiers = get_X_predictions_classifiers(
deep_classifier, rf_classifier, xgboost_classifier, X_test)
X_train_ensemble = df_X_train_preds_classifiers.to_numpy(copy=True)
y_train = y_train.to_numpy(copy=True)
X_test_ensemble = df_X_test_preds_classifiers.to_numpy(copy=True)
y_test = y_test.to_numpy(copy=True)
# train ---------------------------------------------------
#ensemble_classifier = data_science.train_deep_classifier(X_train_ensemble, y_train, X_test_ensemble, y_test, lr=0.0001)
#ensemble_classifier = data_science.train_deep_small_classifier(X_train_ensemble, y_train, X_test_ensemble, y_test, lr=0.0001)
ensemble_classifier = deep_classifiers.load_model_classifier(
'outputs/2019-06-03_14:31:16_model.hdf5')
y_pred_ensemble = deep_classifiers.predict(ensemble_classifier,
X_test_ensemble)
y_pred_deep = deep_classifiers.predict(deep_classifier, X_test)
y_pred_rf = sk_learn_classifiers.predict(rf_classifier, X_test)
y_pred_xgboost = sk_learn_classifiers.predict(xgboost_classifier, X_test)
utils.test_model(y_test, y_pred_deep)
utils.test_model(y_test, y_pred_rf)
utils.test_model(y_test, y_pred_xgboost)
utils.test_model(y_test, y_pred_ensemble)
import sklearn.model_selection as sk
import tensorflow as tf
from tensorflow.keras import Sequential
from tensorflow.keras.layers import Conv2D, BatchNormalization, Dropout, Flatten, Dense, MaxPool2D
import cv2
from utils import load_images, output_model, enable_cuda, test_model, load_model, get_char
images, labels = load_images()
model = load_model("model1")
print(test_model(model, "hello_world.png", True))
x_train, x_test, y_train, y_test = sk.train_test_split(images,
labels,
test_size=.15)
model = tf.keras.Sequential()
model.add(Conv2D(32, kernel_size=(3, 3)))
model.add(Conv2D(64, kernel_size=(4, 4)))
model.add(BatchNormalization())
model.add(MaxPool2D(pool_size=(3, 3)))
model.add(Dropout(0.1))
model.add(Conv2D(128, kernel_size=(5, 5), strides=(2, 2), padding="same"))
model.add(BatchNormalization())
model.add(Flatten())
model.add(Dense(2048))
model.add(Dropout(0.1))
model.add(Dense(1024))
model.add(Dense(62, activation="softmax"))
from utils import append_timestamp, test_model
if __name__ == '__main__':
report_folder = "../reports/"
model_folder = "../models/"
models = {
"AE_classifier_29-Mar-2019__17-49-01.hdf5": {
"report": "AE"
},
"classifier_DANN_final_07-Apr-2019__00-43-33.hdf5": {
"report": "DANN"
},
"SDAE_classifier_06-Apr-2019__15-51-40.hdf5": {
"report": "SDAE"
}
}
data = {
"source": "../data/test_movies_vectors_balanced.csv",
"target": "../data/test_electr_vectors_balanced.csv"
}
for model_name in models:
model_path = model_folder + model_name
model = load_model(model_path)
for k in data:
report = "{}_test_{}_V2.csv".format(k, models[model_name]["report"])
report_path = append_timestamp(report_folder + report)
comment = "Testing {} on {} domain with new predict processing"
comment = comment.format(model_name, k)
test_model(model, data[k], report_path, 1000, comment=comment, norm=True)
('drop2',nn.Dropout(0.2)),
('h3', nn.Linear(512, 133)),
('output', nn.LogSoftmax(dim=1))
]))
model_transfer.fc = classifier
if use_cuda:
model_transfer = model_transfer.cuda()
# Set optimization criterion
criterion_transfer = nn.NLLLoss()
optimizer_transfer = optim.RMSprop(model_transfer.fc.parameters(), lr=0.001)
loaders_transfer = {'train': trainLoad,'val': valLoad,'test': testLoad}
model_transfer = train_model(35, loaders_transfer, model_transfer, optimizer_transfer,
criterion_transfer, use_cuda, 'model_transfer_best.pt')
# Test the model using the testing set
model_transfer.load_state_dict(torch.load('model_transfer_best.pt'))
test_model(loaders_transfer, model_transfer, criterion_transfer, use_cuda)
# Save the classes
class_names = [item[4:].replace("_", " ") for item in train.classes]
class_numbers = [item_element for item_element,item in enumerate(train.classes)]
# Save the class names and indices
with open('Dog_names.pkl', 'wb') as handle:
pkl.dump(class_names, handle)
with open('Dog_indices.pkl', 'wb') as handle:
pkl.dump(class_numbers, handle)
def main(args):
# Generate data
generate_data = get_data_generator(
N=args.n_samples,
mode=None) # Not applying over or under sampling technique
train_loader, test_loader = generate_data()
# Choose device to use
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print(f"Using {device}")
# Define criterion to optimize
if args.model_version == 4:
criterion = digit_prediction_criterion
else:
criterion = get_auxiliary_loss_model_criterion()
# Grid Search
if args.grid_search:
print("Grid Search of learning rate and regularization term")
model_class = NetSiamese
model_params = {
"input_channels": INPUT_CHANNELS,
"output_class_channels": OUTPUT_CLASS_CHANNELS,
"output_digit_channels": OUTPUT_DIGIT_CHANNELS,
"activation": "leakyrelu",
"auxiliary_loss": True,
"version": args.model_version
}
lr, reg = grid_search([0.05, 0.01, 0.005, 0.001, 0.0005, 0.0001],
[0.25, 0.1, 0.05, 0.01],
train,
train_loader,
test_loader,
device,
model_class,
model_params,
criterion,
print_info=True)
else:
lr, reg = args.lr, args.reg
# Train
print(f"Training Siamese model version {args.model_version}" +
(f" ({args.n_rounds} rounds)" if args.n_rounds > 1 else ""))
if args.n_rounds > 1:
model_class = NetSiamese
model_params = {
"input_channels": INPUT_CHANNELS,
"output_class_channels": OUTPUT_CLASS_CHANNELS,
"output_digit_channels": OUTPUT_DIGIT_CHANNELS,
"activation": "leakyrelu",
"auxiliary_loss": True,
"version": args.model_version
}
accuracy_values, loss_values = test_model(train,
generate_data,
device,
model_class,
model_params,
criterion,
lr,
reg,
nb_tests=args.n_rounds,
epochs=args.n_epoch)
if args.plot_curves:
plot_test_results(
accuracy_values,
loss_values,
title=f"Model's assessment over {args.n_rounds} rounds")
print(
f"Mean accuracy is {np.mean(np.max(accuracy_values, axis=1)):0.3f}, "
f"Standard deviation of accuracy is {np.std(np.max(accuracy_values, axis=1)):0.3f}"
)
else:
net_auxiliary_loss = NetSiamese(INPUT_CHANNELS,
OUTPUT_CLASS_CHANNELS,
OUTPUT_DIGIT_CHANNELS,
activation="leakyrelu",
auxiliary_loss=True,
version=args.model_version)
optimizer = optim.Adam(net_auxiliary_loss.parameters(),
lr=lr,
weight_decay=reg)
# Train model
_, accuracies, losses = train(train_loader,
test_loader,
net_auxiliary_loss,
optimizer,
criterion,
device=device,
epochs=args.n_epoch,
print_info=True)
accuracy_train_class, accuracy_test_class, accuracy_train_digit, accuracy_test_digit = accuracies
if args.plot_curves:
plot_accuracy_and_loss(accuracy_train_class,
accuracy_test_class,
losses,
title="Class prediction accuracy")
def _train(model, optim_inf, scheduler_inf, checkpointer, epochs, train_loader,
test_loader, stat_tracker, log_dir, device, args):
'''
Training loop for optimizing encoder
'''
# If mixed precision is on, will add the necessary hooks into the model
# and optimizer for half() conversions
model, optim_inf = mixed_precision.initialize(model, optim_inf)
optim_raw = mixed_precision.get_optimizer(optim_inf)
# get target LR for LR warmup -- assume same LR for all param groups
for pg in optim_raw.param_groups:
lr_real = pg['lr']
# IDK, maybe this helps?
# but it makes the training slow
# torch.cuda.empty_cache()
# prepare checkpoint and stats accumulator
next_epoch, total_updates = checkpointer.get_current_position()
fast_stats = AverageMeterSet()
# run main training loop
for epoch in range(next_epoch, epochs):
epoch_stats = AverageMeterSet()
epoch_updates = 0
time_start = time.time()
for _, ((images1, images2), labels) in enumerate(train_loader):
# get data and info about this minibatch
labels = torch.cat([labels, labels]).to(device)
images1 = images1.to(device)
images2 = images2.to(device)
# run forward pass through model to get global and local features
res_dict = model(args, x1=images1, x2=images2, class_only=False)
lgt_glb_mlp, lgt_glb_lin = res_dict['class']
# compute costs for all self-supervised tasks
loss_g2l = (res_dict['g2l_1t5'] + res_dict['g2l_1t7'] +
res_dict['g2l_5t5'])
loss_inf = loss_g2l + res_dict['lgt_reg']
# compute loss for online evaluation classifiers
loss_cls = (loss_xent(lgt_glb_mlp, labels) +
loss_xent(lgt_glb_lin, labels))
# do hacky learning rate warmup -- we stop when LR hits lr_real
if (total_updates < 500):
lr_scale = min(1., float(total_updates + 1) / 500.)
for pg in optim_raw.param_groups:
pg['lr'] = lr_scale * lr_real
# reset gradient accumlators and do backprop
loss_opt = loss_inf + loss_cls
optim_inf.zero_grad()
mixed_precision.backward(
loss_opt, optim_inf) # backwards with fp32/fp16 awareness
if args.grad_clip_value and (total_updates >= 500):
torch.nn.utils.clip_grad_value_(model.parameters(),
args.grad_clip_value)
optim_inf.step()
# record loss and accuracy on minibatch
epoch_stats.update_dict(
{
'loss_inf': loss_inf.detach().item(),
'loss_cls': loss_cls.detach().item(),
'loss_g2l': loss_g2l.detach().item(),
'lgt_reg': res_dict['lgt_reg'].detach().item(),
'loss_g2l_1t5': res_dict['g2l_1t5'].detach().item(),
'loss_g2l_1t7': res_dict['g2l_1t7'].detach().item(),
'loss_g2l_5t5': res_dict['g2l_5t5'].detach().item()
},
n=1)
update_train_accuracies(epoch_stats, labels, lgt_glb_mlp,
lgt_glb_lin)
# shortcut diagnostics to deal with long epochs
total_updates += 1
epoch_updates += 1
# this command makes the training slow
# torch.cuda.empty_cache()
if (total_updates % 100) == 0:
# IDK, maybe this helps?
time_stop = time.time()
spu = (time_stop - time_start) / 100.
print(
'Epoch {0:d}, {1:d} updates -- {2:.4f} sec/update'.format(
epoch, epoch_updates, spu))
time_start = time.time()
if (total_updates % 500) == 0:
# record diagnostics
eval_start = time.time()
fast_stats = AverageMeterSet()
test_model(args,
model,
test_loader,
device,
fast_stats,
max_evals=100000)
stat_tracker.record_stats(
fast_stats.averages(total_updates, prefix='fast/'))
eval_time = time.time() - eval_start
stat_str = fast_stats.pretty_string(ignore=model.tasks)
stat_str = '-- {0:d} updates, eval_time {1:.2f}: {2:s}'.format(
total_updates, eval_time, stat_str)
print(stat_str)
# update learning rate
scheduler_inf.step(epoch)
test_model(args,
model,
test_loader,
device,
epoch_stats,
max_evals=500000)
epoch_str = epoch_stats.pretty_string(ignore=model.tasks)
diag_str = '{0:d}: {1:s}'.format(epoch, epoch_str)
print(diag_str)
sys.stdout.flush()
stat_tracker.record_stats(epoch_stats.averages(epoch, prefix='costs/'))
checkpointer.update(epoch + 1, total_updates)
# X = torch.sqrt(X + 1e-5)
X = torch.nn.functional.normalize(X)
X = self.fc(X)
assert X.size() == (N, 11)
return X
data_transform = transforms.Compose([
transforms.Resize(448),
# transforms.CenterCrop(224),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])])
# data_dir = '/home/mcc/data/xuelang'
data_dir = '../data/512'
image_dataset = datasets.ImageFolder(os.path.join(data_dir, 'test_r2'), data_transform)
dataloader = torch.utils.data.DataLoader(image_dataset, batch_size=32,
shuffle=False, num_workers=12)
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
model_ft = BCNN()
model_ft.load_state_dict(torch.load('../output/bcnn_step2_20180830_0953.pth'))
model_ft = model_ft.to(device)
all_prob = test_model(model_ft, dataloader, device)
img_names = sorted(os.listdir(os.path.join(data_dir, 'test_r2', 'norm')))
# submit_csv('bcnn', img_names, all_prob)
with open('../output/prob_bcnn.pkl', 'wb') as f:
pickle.dump(all_prob, f)
W_new_new = new_W.reshape(Wshape)
new_layer = Convolution1D(output_dim,
f_dim[1],
strides=1,
activation=layer.activation,
padding='valid',
weights=[new_W, b],
name='converted_conv')
flattened_ipt = False
else:
shape = (1, 1, input_shape[1], output_dim)
new_W = W.reshape(shape)
new_layer = Convolution1D(output_dim, (1, 1),
strides=(1, 1),
activation=layer.activation,
padding='valid',
weights=[new_W, b])
else:
new_layer = layer
new_model.add(new_layer)
new_model.add(Lambda(lambda x: K.batch_flatten(x)))
X, Y = utils.load_data('Dataframes/Testing24.pickle')
score = utils.test_model(new_model, X, Y)
print("%s: %.2f%%" % (new_model.metrics_names[1], score))
memlog_fname=mem_logfile_AE,
epochs=epochs, ae=True)
model.layers.pop()
model.save("../models/AE_layers_popped_{}.hdf5".format(timestamp))
hidden_size1 = 64
hidden_size2 = 128
model.add(LSTM(hidden_size1, return_sequences=True, input_shape=(None, 128)))
# model.add(LSTM(hidden_size2, return_sequences=True))
# model.add(Dense(hidden_size2, activation="hard_sigmoid"))
model.add(Dense(1, activation='hard_sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam')
steps_per_epoch = int(movies_lines / batch_size)
epochs = 20
log_fname = '{}/training_AE_classifier.csv'.format(report_folder)
print("Training classifier")
train_model(model,
train_path["movies"],
batch_size=batch_size,
steps_per_epoch=steps_per_epoch,
log_fname=log_fname,
memlog_fname=mem_logfile_classifier,
epochs=epochs)
print("Testing model")
report_path = "{}/report_LSTM_AE_source.csv".format(report_folder)
test_model(model, test_path["movies"], report_path, batch_size)
report_path = "{}/report_LSTM_AE_target.csv".format(report_folder)
test_model(model, test_path["electr"], report_path, batch_size)
model.save("../models/LSTM_AE_{}_{}.hdf5".format(hidden_size1, timestamp))
print("Done!")
# Command Line ardguments
ap.add_argument('--engine', default="cuda")
ap.add_argument('--hidden_layers', type=int, nargs='+', default=[4096,2048, 512], help='Number of units per hidden layer')
ap.add_argument('--output_size', type=int, default=102, help='Number of possible classifications')
ap.add_argument('--lr', type=float, action="store", default=0.001)
ap.add_argument('--dropout', type=float, action = "store", default = 0.25)
ap.add_argument('--epochs', action="store", type=int, default = 1)
ap.add_argument('--arch', action="store", default="vgg19", type = str)
ap.add_argument('--print_every', action='store', default=10 )
ap.add_argument('--input_dir', nargs='*', action="store", default="flowers")
ap.add_argument('--save_dir', dest="save_dir", action="store", default="./checkpoint.pth")
args = ap.parse_args()
train_dataloader, val_dataloader, test_dataloader, class_to_idx_train = utils.load_data(args.input_dir)
model, criterion, optimizer = utils.build_model(args.arch,args.hidden_layers,args.output_size,args.dropout,args.lr, args.engine)
utils.train_model(model, train_dataloader, val_dataloader, criterion, optimizer, args.epochs, args.print_every, args.engine)
utils.test_model(model, test_dataloader)
utils.save_model(model, args.epochs, optimizer, class_to_idx_train)
print("All Done. The Model is trained")
def GAN():
# Graph Part #
print("Graph initialization...")
with tf.device(FLAGS.device):
with tf.variable_scope("model", reuse=None):
m_train = G.BEGAN(batch_size=FLAGS.tr_batch_size,
is_training=True,
num_keys=FLAGS.num_keys,
input_length=FLAGS.hidden_state_size,
output_length=FLAGS.predict_size,
learning_rate=learning_rate)
with tf.variable_scope("model", reuse=True):
m_valid = G.BEGAN(batch_size=FLAGS.val_batch_size,
is_training=False,
num_keys=FLAGS.num_keys,
input_length=FLAGS.hidden_state_size,
output_length=FLAGS.predict_size,
learning_rate=learning_rate)
with tf.variable_scope("model", reuse=True):
m_test = G.BEGAN(batch_size=FLAGS.test_batch_size,
is_training=False,
num_keys=FLAGS.num_keys,
input_length=FLAGS.hidden_state_size,
output_length=FLAGS.predict_size,
learning_rate=learning_rate)
print("Done")
# Summary Part #
print("Setting up summary op...")
g_loss_ph = tf.placeholder(dtype=tf.float32)
d_loss_ph = tf.placeholder(dtype=tf.float32)
loss_summary_op_d = tf.summary.scalar("discriminatr_loss", d_loss_ph)
loss_summary_op_g = tf.summary.scalar("generator_loss", g_loss_ph)
valid_summary_writer = tf.summary.FileWriter(logs_dir + '/valid/',
max_queue=2)
train_summary_writer = tf.summary.FileWriter(logs_dir + '/train/',
max_queue=2)
print("Done")
# Model Save Part #
print("Setting up Saver...")
saver = tf.train.Saver()
ckpt = tf.train.get_checkpoint_state(logs_dir)
print("Done")
# Session Part #
print("Setting up Data Reader...")
validation_dataset_reader = mt.Dataset(
directory=test_dir,
batch_size=FLAGS.val_batch_size,
is_batch_zero_pad=FLAGS.is_batch_zero_pad,
hidden_state_size=FLAGS.hidden_state_size,
predict_size=FLAGS.predict_size,
num_keys=FLAGS.num_keys,
tick_interval=tick_interval,
step=FLAGS.slice_step)
test_dataset_reader = mt.Dataset(directory=test_dir,
batch_size=FLAGS.test_batch_size,
is_batch_zero_pad=FLAGS.is_batch_zero_pad,
hidden_state_size=FLAGS.hidden_state_size,
predict_size=FLAGS.predict_size,
num_keys=FLAGS.num_keys,
tick_interval=tick_interval,
step=FLAGS.slice_step)
print("done")
sess_config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)
sess_config.gpu_options.allow_growth = True
sess = tf.Session(config=sess_config)
if ckpt and ckpt.model_checkpoint_path: # model restore
saver.restore(sess, ckpt.model_checkpoint_path)
print("Model restored...")
else:
sess.run(tf.global_variables_initializer()
) # if the checkpoint doesn't exist, do initialization
if FLAGS.mode == "train":
train_dataset_reader = mt.Dataset(
directory=train_dir,
batch_size=FLAGS.tr_batch_size,
is_batch_zero_pad=FLAGS.is_batch_zero_pad,
hidden_state_size=FLAGS.hidden_state_size,
predict_size=FLAGS.predict_size,
num_keys=FLAGS.num_keys,
tick_interval=tick_interval,
step=FLAGS.slice_step)
for itr in range(MAX_EPOCH):
feed_dict = utils.run_epoch(train_dataset_reader,
FLAGS.tr_batch_size, m_train, sess)
if itr % 100 == 0:
if FLAGS.use_began_loss:
train_loss_d, train_loss_g, train_pred = sess.run(
[m_train.loss_d, m_train.loss_g, m_train.predict],
feed_dict=feed_dict)
train_summary_str_d, train_summary_str_g = sess.run(
[loss_summary_op_d, loss_summary_op_g],
feed_dict={
g_loss_ph: train_loss_g,
d_loss_ph: train_loss_d
})
train_summary_writer.add_summary(train_summary_str_g, itr)
print("Step : %d TRAINING LOSS *****************" % (itr))
print("Dicriminator_loss: %g\nGenerator_loss: %g" %
(train_loss_d, train_loss_g))
if itr % 1000 == 0:
if FLAGS.use_began_loss:
valid_loss_d, valid_loss_g, valid_pred = utils.validation(
validation_dataset_reader, FLAGS.val_batch_size,
m_valid, FLAGS.hidden_state_size, FLAGS.predict_size,
sess, logs_dir, itr, tick_interval)
valid_summary_str_d, valid_summary_str_g = sess.run(
[loss_summary_op_d, loss_summary_op_g],
feed_dict={
g_loss_ph: valid_loss_g,
d_loss_ph: valid_loss_d
})
valid_summary_writer.add_summary(valid_summary_str_d, itr)
print("Step : %d VALIDATION LOSS ***************" % (itr))
print("Dicriminator_loss: %g\nGenerator_loss: %g" %
(valid_loss_d, valid_loss_g))
if itr % 1000 == 0 and itr != 0:
utils.test_model(test_dataset_reader, FLAGS.test_batch_size,
m_test, FLAGS.predict_size, sess, logs_dir,
itr, tick_interval, 5)
if itr % 1000 == 0:
saver.save(sess, logs_dir + "/model.ckpt", itr)
if FLAGS.mode == "test":
utils.test_model(test_dataset_reader, FLAGS.test_batch_size, m_test,
FLAGS.predict_size, sess, logs_dir, 9999,
tick_interval, 10)
validation_split=0.3,
shuffle=True,
callbacks=[reduce_lr, model_checkpoint, early_stopping])
# print('predict test data')
#
# imgs_mask_test = model.predict({'inputs_modality1': imgs_test, 'inputs_modality2': imgs_test, 'inputs_modality3': imgs_test},
# batch_size=1, verbose=1)
#
# np.save(save_path+'/imgs_mask_test.npy', imgs_mask_test)
def save_img(self):
print("array to image")
imgs = np.load(save_path + '/imgs_mask_test.npy')
for i in range(imgs.shape[0]):
#img = imgs[i]
img = np.squeeze(imgs[i]) * 255
img = np.clip(img, 0, 255).astype('uint8')
save_images(os.path.join(save_path, '%d.jpg' % (i)), img)
#img = array_to_img(img)
#img.save("./results/%d.jpg"%(i))
if __name__ == '__main__':
myunet = myUnet()
myunet.train()
# myunet.save_img()
print('predict test model')
test_model(myunet)
self.save(fname)
print('Training completed in', int((time.time() - start_time) / 60), "minutes &", int(duration % 60), "seconds")
#@tf.function
def train_step(self, source, optimizer):
with tf.GradientTape() as tape:
w, w_mean, w_log_var = self.encode(source)
prediction = self(source[0])
loss_identity = identity_lr * tf.reduce_mean(tf.reduce_sum(cross_entropy(source[0], prediction))) #+ 0.1 * tf.reduce_mean((source[0]-prediction)**2))
kl_loss = -0.5 * (1 + w_log_var - tf.square(w_mean) - tf.exp(w_log_var))
kl_loss = kl_lr * tf.reduce_mean(tf.reduce_sum(kl_loss, axis=1))
loss = (loss_identity + kl_loss) * learning_rate
grads = tape.gradient(loss, model.trainable_variables)
optimizer.apply_gradients(zip(grads, model.trainable_variables))
return loss_identity, kl_loss
def build_graph(self):
source = Input(shape=(256, 256, 3))
return tf.keras.Model(inputs=source, outputs=self.call(source), name=model_name)
model = AE_A()
tf.keras.utils.plot_model(model.build_graph(), model_name+".png", show_shapes=True, expand_nested=True)
model.train_model(model_name)
model.load(model_name)
image_dataset, _ = utils.create_dataset(batch_size=batch_size)
for source, style in zip(image_dataset.take(1), image_dataset.take(1)):
utils.test_model(model, source, style, test=True, name=model_name)
break
def main():
"""Run training."""
config = yaml.safe_load(open("config.yml"))
#training hyperparameters
num_epochs = config['num_epochs']
learning_rate = config['learning_rate']
batch_size = config['batch_size']
valid_period = config['valid_period']
#data hyperparameters
#extract sample (input sequences with length = stride_len)
#for RNN from the givent full trajectory
window_len = config['window_len']
stride_len = config['stride_len']
n_times = config['n_times']
x_train, y_train, x_dev, y_dev, x_test, y_test = read_data(
window_len, stride_len, n_times)
print('Dataset:', x_train.shape, y_train.shape, x_dev.shape, y_dev.shape,
x_test.shape, y_test.shape)
#create dataset
transformed_control_dataset_train = ControlDataset(
x_train, y_train, transform=transforms.Compose([ToTensor()]))
transformed_control_dataset_dev = ControlDataset(
x_dev, y_dev, transform=transforms.Compose([ToTensor()]))
#transformed_control_dataset_test = ControlDataset(x_test, y_test,
#transform=transforms.Compose([ToTensor()]))
# create batch
data_train = DataLoader(transformed_control_dataset_train,
batch_size=batch_size,
shuffle=True,
num_workers=1,
drop_last=True)
data_dev = DataLoader(transformed_control_dataset_dev,
batch_size=batch_size,
shuffle=False,
num_workers=1,
drop_last=True)
#data_test = DataLoader(transformed_control_dataset_test, batch_size=1,
#shuffle=False, num_workers=1, drop_last=True)
# Device
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print('Device:', device)
# save model
save_model_name = config['model_name']
print('Save Model name: ', save_model_name)
model_save_path = '../checkpoints/' + save_model_name #+ '.pt' ## not pth
#test/train
mode = int(sys.argv[1])
if mode == 0:
#train mode
# Model preparation
model = SeRNN_FWXX(batch_size, device)
print('Model: ', model)
model.to(device)
pytorch_total_params = sum(p.numel() for p in model.parameters())
print('# of params: ', pytorch_total_params)
# multiple GPUs
if torch.cuda.device_count() > 1:
print("Model uploaded. Number of GPUs: ",
torch.cuda.device_count())
#model = nn.DataParallel(model)
#set the training loss and optimizer
criterion = nn.MSELoss()
optimizer = optim.RMSprop(model.parameters(), lr=learning_rate)
train_model(model, device, data_train, data_dev, x_dev, y_dev,
optimizer, criterion, num_epochs, model_save_path,
window_len, stride_len, valid_period)
else:
# test mode
# upload saved model
print('Saved Model evaluation with test set')
model = torch.jit.load(model_save_path)
test_model(model, device, x_test, y_test, 'True', window_len,
stride_len)
labels[seq["seqid"]] = label
for label in seqs.keys():
random.shuffle(seqs[label])
errors = []
sizes = []
ratios = [.01, .05, .1, .25, .5, .75, .9];
for ratio in ratios:
training = []
testing = []
for label in seqs.keys():
#random.shuffle(seqs[label]) #So our training sets are not proper subsets of eachother
k = int(ratio * len(seqs[label]))
training.extend(seqs[label][:k])
testing.extend(seqs[label][k:])
print >>sys.stderr, "Ratio %s, training: %s, testing: %s" % (ratio, len(training), len(testing))
sizes.append(len(training))
models = utils.build_models(training, labels)
error, log_odds = utils.test_model(testing, labels, models)
errors.append(error)
#print errors
print sizes
print errors
