def run(self):
try:
if self.load_snapshot:
net = load_model(self.load_snapshot)
else:
from net import ResNet
net = ResNet(N)
net.create()
class PredictStash(object):
def __init__(self, trigger, res_queues):
self.stash = []
self.trigger = trigger
self.res_queues = res_queues
def add(self, kind, X_pos):
self.stash.append((kind, X_pos))
if len(self.stash) >= self.trigger:
self.process()
def process(self):
if not self.stash:
return
value = net.predict([s[1] for s in self.stash])
except:
import traceback
traceback.print_exc()
def __init__(self, model_path=None):
"""
Params:
model_path: Optional pretrained model file
"""
# Initialize model
self.model = ResNet().cuda()
if model_path is not None:
self.model.load_state_dict(torch.load(model_path))
print('Model read from {}.'.format(model_path))
print('Predictor initialized.')
def main():
parser = argparse.ArgumentParser(description='extract features')
parser.add_argument('--gpu', '-g', default=0, type=int, help='GPU ID')
parser.add_argument('--ALL_DATA_PATH', '-all', default='../data/processed_data.pickle')
parser.add_argument('--MODEL_PATH', '-model', default='../data/resnet_152.caffemodel')
parser.add_argument('--SAVE_PATH', '-save', default='../data/all_feature.pickle')
args = parser.parse_args()
gpu_device = args.gpu
all_data_path = args.ALL_DATA_PATH
model_path = args.MODEL_PATH
save_path = args.SAVE_PATH
with open(all_data_path, 'rb') as f:
data = pickle.load(f)
cuda.get_device_from_id(gpu_device).use()
model = ResNet(model_path, 152)
model.to_gpu(gpu_device)
all_dic = {}
for each_question in data:
try:
image_path = each_question['image_path']
image = io.imread(image_path)
if image.ndim == 2:
image = np.tile(image[:, :, np.newaxis], (1, 1, 3))
qa_id = each_question['qa_id']
crop_region = [each_question['x'], each_question['y'], each_question['w'], each_question['h']]
h, w, _ = image.shape
x_region, y_region, w_region, h_region = crop_region
resize_entire = image_resize(image)
resize_region = region_resize(image, crop_region)
entire_feature = feature_extract(resize_entire, model)
region_feature = feature_extract(resize_region, model)
concat_feature = np.concatenate((region_feature, entire_feature), axis=1)[0]
x_tl, y_tl, x_br, y_br = x_region, y_region, x_region + w_region, y_region + h_region
region_inf = np.asarray([x_tl/w, y_tl/h, x_br/w, y_br/h, (w_region * h_region)/(w * h)])
concat_all = np.concatenate((concat_feature, region_inf), axis=0)
all_dic[qa_id] = concat_all
except:
continue
with open(save_path, 'wb') as f:
pickle.dump(all_dic, f)
def __init__(self, model_name, snapshot, num_classes):
self.num_classes = num_classes
if model_name == "resnet50":
self.model = ResNet(torchvision.models.resnet50(pretrained=False),
num_classes)
elif model_name == "mobilenetv2":
self.model = MobileNetV2(num_classes=num_classes)
else:
print("No such model...")
exit(0)
self.saved_state_dict = torch.load(snapshot)
self.model.load_state_dict(self.saved_state_dict)
self.model.cuda(0)
self.model.eval() # Change model to 'eval' mode
self.softmax = nn.Softmax(dim=1).cuda(0)
self.optimizer = Optimize()
def initialize_resnet(self,
ckpt_path,
num_res_layers=None,
num_channels=None,
replace_unloaded_resnet_by_naivenet=True):
""" Initializes a residual network.
Args:
ckpt_path: string: path to the checkpoint to be loaded.
tf_device: string: tensorflow string for tf.device to be used.
num_res_layers: int: number of residual blocks in the CNN.
num_channels: int: number of channels in the CNN layers.
replace_unloaded_resnet_by_naivenet: boolean: if True, when ckpt_path
is None, a NaiveNet is used instead of a ResNet with random
weights.
Returns:
tf.keras.Model: the initialized network.
"""
if ckpt_path is not None or not replace_unloaded_resnet_by_naivenet:
if num_channels is None:
num_channels = self.num_channels
if num_res_layers is None:
num_res_layers = self.num_res_layers
self.net = ResNet(self._num_possible_moves(), num_res_layers,
num_channels)
print('Created resnet with %d res layers and %d channels' %
(num_res_layers, num_channels))
if ckpt_path is not None:
try:
checkpoint = tfe.Checkpoint(net=self.net)
checkpoint.restore(ckpt_path)
print('Loaded resnet params from: ' + ckpt_path)
except tf.errors.NotFoundError:
pass
else:
self.net = NaiveNet(self._num_possible_moves())
print('Created naive net')
class Test:
def __init__(self, model_name, snapshot, num_classes):
self.num_classes = num_classes
if model_name == "resnet50":
self.model = ResNet(torchvision.models.resnet50(pretrained=False),
num_classes)
elif model_name == "mobilenetv2":
self.model = MobileNetV2(num_classes=num_classes)
else:
print("No such model...")
exit(0)
self.saved_state_dict = torch.load(snapshot)
self.model.load_state_dict(self.saved_state_dict)
self.model.cuda(0)
self.model.eval() # Change model to 'eval' mode
self.softmax = nn.Softmax(dim=1).cuda(0)
self.optimizer = Optimize()
def draw_vectors(self, pred_vector1, pred_vector2, pred_vector3, img,
center, width):
optimize_v = self.optimizer.Get_Ortho_Vectors(np.array(pred_vector1),
np.array(pred_vector2),
np.array(pred_vector3))
v1, v2, v3 = optimize_v[0], optimize_v[1], optimize_v[2]
# draw vector in blue color
predx, predy, predz = v1
utils.draw_front(img,
predx,
predy,
width,
tdx=center[0],
tdy=center[1],
size=100,
color=(255, 0, 0))
# draw vector in green color
predx, predy, predz = v2
utils.draw_front(img,
predx,
predy,
width,
tdx=center[0],
tdy=center[1],
size=100,
color=(0, 255, 0))
# draw vector in red color
predx, predy, predz = v3
utils.draw_front(img,
predx,
predy,
width,
tdx=center[0],
tdy=center[1],
size=100,
color=(0, 0, 255))
cv.imshow("pose visualization", img)
def test_per_img(self, cv_img, draw_img, center, w):
with torch.no_grad():
images = cv_img.cuda(0)
# get x,y,z cls predictions
x_v1, y_v1, z_v1, x_v2, y_v2, z_v2, x_v3, y_v3, z_v3 = self.model(
images)
# get prediction vector(get continue value from classify result)
_, _, _, pred_vector1 = utils.classify2vector(
x_v1, y_v1, z_v1, self.softmax, self.num_classes)
_, _, _, pred_vector2 = utils.classify2vector(
x_v2, y_v2, z_v2, self.softmax, self.num_classes)
_, _, _, pred_vector3 = utils.classify2vector(
x_v3, y_v3, z_v3, self.softmax, self.num_classes)
#visualize vectors
self.draw_vectors(pred_vector1[0].cpu().tolist(),
pred_vector2[0].cpu().tolist(),
pred_vector3[0].cpu().tolist(), draw_img, center,
w)
c = (predicted == labels).squeeze()
for i in range(4):
label = labels[i]
class_correct[label] += c[i].item()
class_total[label] += 1
for i in range(10):
print('Accuracy of %5s : %2d %%' %
(classes[i], 100 * class_correct[i] / class_total[i]))
if __name__ == '__main__':
# Loading and normalizing CIFAR10
trainloader, testloader, classes = data_prepare()
# Define a Convolutional Neural Network
net = ResNet()
net = net.cuda()
x = torch.autograd.Variable(torch.rand(64, 3, 32, 32))
writer.add_graph(net, x.cuda(), verbose=True)
if args.load_model == True:
net.load_state_dict(torch.load(args.model_path))
# Define a Loss function and optimizer
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(net.parameters(),
lr=1e-4,
betas=(0.9, 0.999),
eps=1e-08,
weight_decay=0.,
amsgrad=True)
# Train the network
for epoch in range(args.epochs):
def train_and_valid(learning_rate=lr,
weight_decay=weight_decay,
num_of_res=num_of_res_blocks,
if_bottleneck=bottle,
plot=True):
"""
Train the model and run it on the valid set every epoch
:param weight_decay: for L2 regularzition
:param bottleneck:
:param num_of_res:
:param learning_rate: lr
:param plot: draw the train/valid loss curve or not
:return:
"""
curr_lr = learning_rate
# model define
if NET_TYPE == 'res':
if DATA_TYPE == 'hoa':
block = ResBlock(128, 128, bottleneck=if_bottleneck)
else:
block = ResBlock(256, 256, bottleneck=if_bottleneck)
model = ResNet(block,
numOfResBlock=num_of_res,
input_shape=input_shape,
data_type=DATA_TYPE).to(device)
elif NET_TYPE == 'hoa':
model = HOANet(input_shape=input_shape).to(device)
else:
raise RuntimeError('Unrecognized net type!')
# print(model)
criterion = nn.MSELoss()
optimizer = torch.optim.Adam(model.parameters(),
lr=learning_rate,
weight_decay=weight_decay)
# These parameters are for searching the best epoch to early stopping
train_loss_curve, valid_loss_curve = [], []
best_loss, avr_valid_loss = 10000.0, 0.0
best_epoch = 0
best_model = None # the best parameters
for epoch in range(num_epochs):
# 每一轮的 训练集/验证集 误差
train_loss_per_epoch, valid_loss_per_epoch = 0.0, 0.0
train_step_cnt, valid_step_cnt = 0, 0
train_data, valid_data = [], []
# 进入训练模式
model.train()
random.shuffle(train_file_order)
for idx, train_idx in enumerate(train_file_order):
if len(train_data) < batch_size:
train_data_temp = HOADataSet(
path=DATA_PATH + ('' if DATA_TYPE == 'hoa' else 'STFT/') +
'tr/',
index=train_idx + 1,
data_type=DATA_TYPE,
is_speech=SPEECH)
if len(train_data) == 0:
train_data = train_data_temp
else:
train_data += train_data_temp
continue
train_loader = data.DataLoader(dataset=train_data,
batch_size=batch_size,
shuffle=True)
for step, (examples, labels) in enumerate(train_loader):
# if step == 1:
# break
train_step_cnt += 1
# print(train_step_cnt)
examples = examples.float().to(device)
labels = labels.float().to(device)
outputs = model(examples)
train_loss = criterion(outputs, labels)
train_loss_per_epoch += train_loss.item()
# Backward and optimize
optimizer.zero_grad()
train_loss.backward()
optimizer.step()
logger.info(
"Epoch [{}/{}], Step {}, train Loss: {:.4f}".format(
epoch + 1, num_epochs, train_step_cnt,
train_loss.item()))
train_data = HOADataSet(path=DATA_PATH +
('' if DATA_TYPE == 'hoa' else 'STFT/') +
'tr/',
index=train_idx + 1,
data_type=DATA_TYPE,
is_speech=SPEECH)
if plot:
train_loss_curve.append(train_loss_per_epoch / train_step_cnt)
if running_lr and epoch > 1 and (epoch + 1) % 2 == 0:
curr_lr = curr_lr * (1 - decay)
update_lr(optimizer, curr_lr)
# valid every epoch
# 进入验证模式
model.eval()
with torch.no_grad():
for idx, valid_idx in enumerate(valid_file_order):
if len(valid_data) < batch_size:
valid_data_temp = HOADataSet(
path=DATA_PATH +
('' if DATA_TYPE == 'hoa' else 'STFT/') + 'cv/',
index=valid_idx + 1,
data_type=DATA_TYPE,
is_speech=SPEECH)
if len(valid_data) == 0:
valid_data = valid_data_temp
else:
valid_data += valid_data_temp
continue
valid_loader = data.DataLoader(dataset=valid_data,
batch_size=batch_size,
shuffle=True)
for step, (examples, labels) in enumerate(valid_loader):
valid_step_cnt += 1
# print(valid_step_cnt)
examples = examples.float().to(device)
labels = labels.float().to(device)
outputs = model(examples)
valid_loss = criterion(outputs, labels)
valid_loss_per_epoch += valid_loss.item()
logger.info(
'The loss for the current batch:{}'.format(valid_loss))
valid_data = HOADataSet(
path=DATA_PATH + ('' if DATA_TYPE == 'hoa' else 'STFT/') +
'cv/',
index=valid_idx + 1,
data_type=DATA_TYPE,
is_speech=SPEECH)
avr_valid_loss = valid_loss_per_epoch / valid_step_cnt
logger.info(
'Epoch {}, the average loss on the valid set: {} '.format(
epoch, avr_valid_loss))
valid_loss_curve.append(avr_valid_loss)
if avr_valid_loss < best_loss:
best_loss = avr_valid_loss
best_epoch, best_model = epoch, model.state_dict()
# end for loop of epoch
torch.save(
{
'epoch': best_epoch,
'state_dict': best_model,
'loss': best_loss,
}, './models/ckpoint_' + CUR_TASK + '_bot' + str(int(if_bottleneck)) +
'_lr' + str(learning_rate) + '_wd' + str(weight_decay) + '_#res' +
str(num_of_res) + '.tar')
logger.info('best epoch:{}, valid loss:{}'.format(best_epoch, best_loss))
if plot:
x = np.arange(num_epochs)
fig, ax = plt.subplots(1, 1)
ax.plot(x, train_loss_curve, 'b', label='Train Loss')
ax.plot(x, valid_loss_curve, 'r', label='Valid Loss')
plt.legend(loc='upper right')
plt.savefig(name + '.jpg')
plt.close()
logger.info(
'\n\n====================Training finished=========================')
logger.info(
'========================Validation on valid set====================')
# 读取checkpoint保存的模型，在验证集上跑一遍，计算准确率和召回率
path = './models/ckpoint_' + name + '.tar'
checkpoint = torch.load(path)
# input_shape = (c.hoa_num * 2, c.frames_per_block+2, c.n_freq)
if DATA_TYPE == 'hoa':
block = ResBlock(128, 128, bottleneck=bottle)
else:
block = ResBlock(256, 256, bottleneck=bottle)
model = ResNet(block,
numOfResBlock=num_of_res_blocks,
input_shape=input_shape,
data_type=DATA_TYPE).to(device)
model.load_state_dict(checkpoint['state_dict'])
criterion = nn.MSELoss()
model.eval()
f = open(name + '.txt', 'w')
file_ = open('anechoic_mono_speech_tt.flist', 'r')
all_test_files = file_.readlines()
with torch.no_grad():
for snr in snr_list:
offset = index_offset_dict[snr]
total_correct = np.zeros(numOfEth) # 每个误差容忍度都对应一个准确个数
total_recall = np.zeros(numOfEth)
total_peaks, total_predict = 0, 0 # 总共的真实峰值数，总共预测出来的峰值数
valid_step_cnt = 0
class ResNetPredictor:
def __init__(self, model_path=None):
"""
Params:
model_path: Optional pretrained model file
"""
# Initialize model
self.model = ResNet().cuda()
if model_path is not None:
self.model.load_state_dict(torch.load(model_path))
print('Model read from {}.'.format(model_path))
print('Predictor initialized.')
def fit(self, train_dataset_path, valid_dataset_path, model_dir, **training_args):
"""
train_dataset_path: The path to the training dataset.pkl
valid_dataset_path: The path to the validation dataset.pkl
model_dir: The directory to save models for each epoch
training_args:
- batch_size
- valid_batch_size
- epoch
- lr
- save_every_epoch
"""
# Set paths
self.train_dataset_path = train_dataset_path
self.valid_dataset_path = valid_dataset_path
self.model_dir = model_dir
Path(self.model_dir).mkdir(parents=True, exist_ok=True)
# Set training params
self.batch_size = training_args['batch_size']
self.valid_batch_size = training_args['valid_batch_size']
self.epoch = training_args['epoch']
self.lr = training_args['lr']
self.save_every_epoch = training_args['save_every_epoch']
self.optimizer = optim.Adam(self.model.parameters(), lr=self.lr)
self.onset_criterion = nn.BCEWithLogitsLoss()
self.offset_criterion = nn.BCEWithLogitsLoss()
self.pitch_criterion = nn.CrossEntropyLoss()
# Read the datasets
print('Reading datasets...')
with open(self.train_dataset_path, 'rb') as f:
self.training_dataset = pickle.load(f)
with open(self.valid_dataset_path, 'rb') as f:
self.validation_dataset = pickle.load(f)
# Setup dataloader and initial variables
self.train_loader = DataLoader(
self.training_dataset,
batch_size=self.batch_size,
num_workers=4,
pin_memory=True,
shuffle=True,
drop_last=True,
)
self.valid_loader = DataLoader(
self.validation_dataset,
batch_size=self.valid_batch_size,
num_workers=4,
pin_memory=True,
shuffle=False,
drop_last=False,
)
start_time = time.time()
training_loss_list = []
valid_loss_list = []
# Start training
self.iters_per_epoch = len(self.train_loader)
for epoch in range(1, self.epoch + 1):
self.model.train()
# Run iterations
total_training_loss = 0
for batch_idx, batch in enumerate(self.train_loader):
self.optimizer.zero_grad()
# Parse batch data
input_tensor = batch[0].permute(0, 2, 1).unsqueeze(1).cuda()
osnet_prob, offset_prob, pitch_class = batch[1][:, 0].float().cuda(), batch[1][:, 1].float().cuda(), batch[1][:, 2].cuda()
# Forward model
onset_logits, offset_logits, pitch_logits = self.model(input_tensor)
# Calculate loss
loss = self.onset_criterion(onset_logits, osnet_prob) \
+ self.offset_criterion(offset_logits, offset_prob) \
+ self.pitch_criterion(pitch_logits, pitch_class)
loss.backward()
self.optimizer.step()
total_training_loss += loss.item()
# Free GPU memory
# torch.cuda.empty_cache()
if epoch % self.save_every_epoch == 0:
# Perform validation
self.model.eval()
with torch.no_grad():
total_valid_loss = 0
for batch_idx, batch in enumerate(self.valid_loader):
# Parse batch data
input_tensor = batch[0].permute(0, 2, 1).unsqueeze(1).cuda()
osnet_prob, offset_prob, pitch_class = batch[1][:, 0].float().cuda(), batch[1][:, 1].float().cuda(), batch[1][:, 2].cuda()
# Forward model
onset_logits, offset_logits, pitch_logits = self.model(input_tensor)
# Calculate loss
loss = self.onset_criterion(onset_logits, osnet_prob) \
+ self.offset_criterion(offset_logits, offset_prob) \
+ self.pitch_criterion(pitch_logits, pitch_class)
total_valid_loss += loss.item()
# Free GPU memory
# torch.cuda.empty_cache()
# Save model
save_dict = self.model.state_dict()
target_model_path = Path(self.model_dir) / 'e_{}'.format(epoch)
torch.save(save_dict, target_model_path)
# Save loss list
training_loss_list.append((epoch, total_training_loss/self.iters_per_epoch))
valid_loss_list.append((epoch, total_valid_loss/len(self.valid_loader)))
# Epoch statistics
print(
'| Epoch [{:4d}/{:4d}] Train Loss {:.4f} Valid Loss {:.4f} Time {:.1f}'.format(
epoch,
self.epoch,
training_loss_list[-1][1],
valid_loss_list[-1][1],
time.time()-start_time,
)
)
# Save loss to file
with open('./plotting/data/loss.pkl', 'wb') as f:
pickle.dump({'train': training_loss_list, 'valid': valid_loss_list}, f)
print('Training done in {:.1f} minutes.'.format((time.time()-start_time)/60))
def _parse_frame_info(self, frame_info):
"""Parse frame info [(onset_probs, offset_probs, pitch_class)...] into desired label format."""
onset_thres = 0.25
offset_thres = 0.25
result = []
current_onset = None
pitch_counter = Counter()
last_onset = 0.0
for idx, info in enumerate(frame_info):
current_time = FRAME_LENGTH*idx + FRAME_LENGTH/2
if info[0] >= onset_thres: # If is onset
if current_onset is None:
current_onset = current_time
last_onset = info[0]
elif info[0] >= onset_thres:
# If current_onset exists, make this onset a offset and the next current_onset
if pitch_counter.most_common(1)[0][0] != 49:
result.append([current_onset, current_time, pitch_counter.most_common(1)[0][0] + 36])
elif len(pitch_counter.most_common(2)) == 2:
result.append([current_onset, current_time, pitch_counter.most_common(2)[1][0] + 36])
current_onset = current_time
last_onset = info[0]
pitch_counter.clear()
elif info[1] >= offset_thres: # If is offset
if current_onset is not None:
if pitch_counter.most_common(1)[0][0] != 49:
result.append([current_onset, current_time, pitch_counter.most_common(1)[0][0] + 36])
elif len(pitch_counter.most_common(2)) == 2:
result.append([current_onset, current_time, pitch_counter.most_common(2)[1][0] + 36])
current_onset = None
pitch_counter.clear()
# If current_onset exist, add count for the pitch
if current_onset is not None:
pitch_counter[info[2]] += 1
return result
def predict(self, test_dataset):
"""Predict results for a given test dataset."""
# Setup params and dataloader
batch_size = 500
test_loader = DataLoader(
test_dataset,
batch_size=batch_size,
pin_memory=False,
shuffle=False,
drop_last=False,
)
# Start predicting
results = []
self.model.eval()
with torch.no_grad():
print('Forwarding model...')
song_frames_table = {}
for batch_idx, batch in enumerate(tqdm(test_loader)):
# Parse batch data
input_tensor = batch[0].unsqueeze(1).cuda()
song_ids = batch[1]
# Forward model
onset_logits, offset_logits, pitch_logits = self.model(input_tensor)
onset_probs, offset_probs, pitch_logits = torch.sigmoid(onset_logits).cpu(), torch.sigmoid(offset_logits).cpu(), pitch_logits.cpu()
# Collect frames for corresponding songs
for bid, song_id in enumerate(song_ids):
frame_info = (onset_probs[bid], offset_probs[bid], torch.argmax(pitch_logits[bid]).item())
song_frames_table.setdefault(song_id, [])
song_frames_table[song_id].append(frame_info)
# Parse frame info into output format for every song
results = {}
for song_id, frame_info in song_frames_table.items():
results[song_id] = self._parse_frame_info(frame_info)
return results
def train(net, bins, alpha, beta, batch_size):
"""
params:
bins: number of bins for classification
alpha: regression loss weight
beta: ortho loss weight
"""
# create model
if net == "resnet50":
model = ResNet(torchvision.models.resnet50(pretrained=True),
num_classes=bins)
lr = args.lr_resnet
else:
model = MobileNetV2(bins)
lr = args.lr_mobilenet
# loading data
logger.logger.info("Loading data".center(100, '='))
train_data_loader = loadData(args.train_data, args.input_size, batch_size,
bins)
valid_data_loader = loadData(args.valid_data, args.input_size, batch_size,
bins, False)
# initialize cls loss function
if args.cls_loss == "KLDiv":
cls_criterion = nn.KLDivLoss(reduction='batchmean').cuda(0)
elif args.cls_loss == "BCE":
cls_criterion = nn.BCELoss().cuda(0)
# initialize reg loss function
reg_criterion = nn.MSELoss().cuda(0)
softmax = nn.Softmax(dim=1).cuda(0)
model.cuda(0)
# training log
logger.logger.info("Training".center(100, '='))
# initialize learning rate and step
lr = lr
step = 0
# validation error
min_avg_error = 1000.
# start training
for epoch in range(args.epochs):
print("Epoch:", epoch)
# learning rate initialization
if net == 'resnet50':
if epoch >= args.unfreeze:
optimizer = torch.optim.Adam(
[{
"params": get_non_ignored_params(model, net),
"lr": lr
}, {
"params": get_cls_fc_params(model),
"lr": lr * 10
}],
lr=args.lr_resnet)
else:
optimizer = torch.optim.Adam(
[{
"params": get_non_ignored_params(model, net),
"lr": lr
}, {
"params": get_cls_fc_params(model),
"lr": lr * 10
}],
lr=args.lr_resnet)
else:
if epoch >= args.unfreeze:
optimizer = torch.optim.Adam(
[{
"params": get_non_ignored_params(model, net),
"lr": lr
}, {
"params": get_cls_fc_params(model),
"lr": lr
}],
lr=args.lr_mobilenet)
else:
optimizer = torch.optim.Adam(
[{
"params": get_non_ignored_params(model, net),
"lr": lr * 10
}, {
"params": get_cls_fc_params(model),
"lr": lr * 10
}],
lr=args.lr_mobilenet)
# reduce lr by lr_decay factor for each epoch
lr = lr * args.lr_decay
print("------------")
for i, (images, cls_v1, cls_v2, cls_v3, reg_v1, reg_v2, reg_v3,
name) in enumerate(train_data_loader):
step += 1
images = images.cuda(0)
# get classified labels
cls_v1 = cls_v1.cuda(0)
cls_v2 = cls_v2.cuda(0)
cls_v3 = cls_v3.cuda(0)
# get continuous labels
reg_v1 = reg_v1.cuda(0)
reg_v2 = reg_v2.cuda(0)
reg_v3 = reg_v3.cuda(0)
# inference
x_pred_v1, y_pred_v1, z_pred_v1, x_pred_v2, y_pred_v2, z_pred_v2, x_pred_v3, y_pred_v3, z_pred_v3 = model(
images)
logits = [
x_pred_v1, y_pred_v1, z_pred_v1, x_pred_v2, y_pred_v2,
z_pred_v2, x_pred_v3, y_pred_v3, z_pred_v3
]
loss, degree_error_v1, degree_error_v2, degree_error_v3 = utils.computeLoss(
cls_v1, cls_v2, cls_v3, reg_v1, reg_v2, reg_v3, logits,
softmax, cls_criterion, reg_criterion, [
bins, alpha, beta, args.cls_loss, args.reg_loss,
args.ortho_loss
])
# backward
grad = [torch.tensor(1.0).cuda(0) for _ in range(3)]
optimizer.zero_grad()
torch.autograd.backward(loss, grad)
optimizer.step()
# save training log and weight
if (i + 1) % 100 == 0:
msg = "Epoch: %d/%d | Iter: %d/%d | x_loss: %.6f | y_loss: %.6f | z_loss: %.6f | degree_error_f:%.3f | degree_error_r:%.3f | degree_error_u:%.3f" % (
epoch, args.epochs, i + 1, len(train_data_loader.dataset)
// batch_size, loss[0].item(), loss[1].item(),
loss[2].item(), degree_error_v1.item(),
degree_error_v2.item(), degree_error_v3.item())
logger.logger.info(msg)
# Test on validation dataset
error_v1, error_v2, error_v3 = valid(model, valid_data_loader, softmax,
bins)
print("Epoch:", epoch)
print("Validation Error:", error_v1.item(), error_v2.item(),
error_v3.item())
logger.logger.info("Validation Error(l,d,f)_{},{},{}".format(
error_v1.item(), error_v2.item(), error_v3.item()))
# save model if achieve better validation performance
if error_v1.item() + error_v2.item() + error_v3.item() < min_avg_error:
min_avg_error = error_v1.item() + error_v2.item() + error_v3.item()
print("Training Info:")
print("Model:", net, " ", "Number of bins:", bins, " ", "Alpha:",
alpha, " ", "Beta:", beta)
print("Saving Model......")
torch.save(
model.state_dict(),
os.path.join(snapshot_dir, output_string + '_Best_' + '.pkl'))
print("Saved")
def main(config, resume, phase):
# Dataset
fine_dataset = fine_clustering_dataset(config)
# Dataloder
train_loader = DataLoader(fine_dataset,
shuffle=True,
batch_size=config['batch_size'],
num_workers=32)
val_loader = DataLoader(fine_dataset,
shuffle=False,
batch_size=config['batch_size'],
num_workers=32)
test_loader = DataLoader(fine_dataset,
shuffle=False,
batch_size=config['batch_size'],
num_workers=32)
# Model
start_epoch = 0
if config['model_name'].startswith('resnet'):
model = ResNet(config)
elif config['model_name'].startswith('densenet'):
model = DenseNet(config)
elif config['model_name'].startswith('deeplab'):
cluster_vector_dim = config['cluster_vector_dim']
model = DeepLabv3_plus(nInputChannels=3,
n_classes=3,
os=16,
cluster_vector_dim=cluster_vector_dim,
pretrained=True,
_print=True)
elif config['model_name'].startswith('bagnet'):
model = BagNet(config=config)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
if resume:
filepath = config['pretrain_path']
start_epoch, learning_rate, optimizer, M, s = load_ckpt(
model, filepath)
start_epoch += 1
if torch.cuda.device_count() > 1:
print("Let's use", torch.cuda.device_count(), "GPUs!")
model = nn.DataParallel(model)
model.to(device)
#Optimizer
optimizer = torch.optim.Adam(model.parameters(),
lr=config['learning_rate'],
weight_decay=1e-5)
#resume or not
if start_epoch == 0:
print("Grand New Training")
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
optimizer, patience=config['switch_learning_rate_interval'])
# log_dir = config['log_dir']+"/{}_{}_".format(config['date'],config['model_name'])+"ep_{}-{}_lr_{}".format(start_epoch,start_epoch+config['num_epoch'],config['learning_rate'])
# best loss
if not resume:
learning_rate = config['learning_rate']
M, s = cluster_initialization(train_loader, model, config, phase)
print(start_epoch)
if config['if_train']:
for epoch in range(start_epoch + 1,
start_epoch + config['num_epoch'] + 1):
loss_tr = train(
train_loader, model, optimizer, epoch, config, M,
s) #if training, delete learning rate and add optimizer
if config['if_valid'] and epoch % config[
'valid_epoch_interval'] == 0:
with torch.no_grad():
loss_val, M, s = valid(val_loader, model, epoch, config,
learning_rate, M, s, phase)
scheduler.step(loss_val)
save_ckpt(model, optimizer, epoch, loss_tr, loss_val, config,
M, s)
else:
val_log = open("../log/val_" + config['date'] + ".txt", "a")
val_log.write('epoch ' + str(epoch) + '\n')
val_log.close()
test(test_loader, model, config, M, phase)
store_config(config, phase)
print("Training finished ...")
print("analysis collect score")
utils.collect_score(loss_dict,
os.path.join(args.save_dir, "collect_score"))
if __name__ == '__main__':
args = parse_args()
utils.mkdir(args.save_dir)
# cls and sord
print("Creating model......")
if args.model_name == "mobilenetv2":
model = MobileNetV2(num_classes=args.num_classes)
else:
model = ResNet(torchvision.models.resnet50(pretrained=False),
args.num_classes)
print("Loading weight......")
saved_state_dict = torch.load(args.snapshot)
model.load_state_dict(saved_state_dict)
model.cuda(0)
model.eval() # Change model to 'eval' mode (BN uses moving mean/var).
softmax = nn.Softmax(dim=1).cuda(0)
# test dataLoader
test_loader = loadData(args.test_data, args.input_size, args.batch_size,
args.num_classes, False)
# testing
@author: ashli """ from load import loadim, loadcsv, showgray, ImageToMask #from load import loadb #import chainer #import chainer.functions as F #import chainer.links as L from chainer import Variable, cuda from net import ResNet import numpy as np from train import Trainer import pandas as pd model = ResNet() #gpu_device=0 #cuda.get_device(gpu_device).use() #model.to_gpu(gpu_device) tr_im = loadim(a=-1, j="trainimages") tr_ma = loadim(a=-1, j="trainmasks") #teimage=loadim(a=-1,j="test") #trdepth,tedepth=loadcsv() #trdepth=sorted(trdepth,key=lambda t:t[0]) #tedepth=sorted(tedepth,key=lambda t:t[0]) #tr_dep=[int(t[1]) for t in trdepth] #te_dep=[int(t[1]) for t in tedepth] #tr_im_ma_dep=[[tr_im[i],tr_ma[i],tr_dep[i]] for i in range(4000)] #tr_im_ma_dep=sorted(tr_im_ma_dep,key=lambda t:t[2])
def train(img_dir, xml_dir, epochs, input_size, batch_size, num_classes):
"""
params:
bins: number of bins for classification
alpha: regression loss weight
beta: ortho loss weight
"""
# create model
model = ResNet(torchvision.models.resnet50(pretrained=True),
num_classes=num_classes)
cls_criterion = nn.CrossEntropyLoss().cuda(1)
softmax = nn.Softmax(dim=1).cuda(1)
model.cuda(1)
# initialize learning rate and step
lr = 0.001
step = 0
optimizer = torch.optim.Adam(model.parameters(), lr=lr)
#load data
train_data_loader = loadData(img_dir, xml_dir, input_size, batch_size,
True)
test_loader = loadData('../yolov3/data/test_imgs',
'../yolov3/data/test_anns', 224, 8, False)
#variables
history = []
best_acc = 0.0
best_epoch = 0
# start training
for epoch in range(epochs):
print("Epoch:", epoch)
print("------------")
# reduce lr by lr_decay factor for each epoch
if epoch % 10 == 0:
lr = lr * 0.9
train_loss = 0.0
train_acc = 0
val_acc = 0
model.train()
for i, (images, labels) in enumerate(train_data_loader):
if i % 10 == 0:
print("batch: {}/{}".format(
i,
len(train_data_loader.dataset) // batch_size))
images = images.cuda(1)
labels = labels.cuda(1)
# backward
optimizer.zero_grad()
outputs = model(images)
loss = cls_criterion(outputs, labels)
loss.backward()
optimizer.step()
train_loss += loss.item()
ret, predictions = torch.max(outputs.data, 1)
correct_counts = predictions.eq(labels.data.view_as(predictions))
acc = torch.mean(correct_counts.type(torch.FloatTensor))
train_acc += acc.item() * images.size(0)
print("epoch: {:03d}, Training loss: {:.4f}, Accuracy: {:.4f}%".format(
epoch + 1, train_loss, train_acc / 3096 * 100))
#if (epoch+1) % 3 == 0:
# torch.save(model, 'models/'+'model_'+str(epoch+1)+'.pt')
print("Start testing...")
with torch.no_grad():
model.eval()
for j, (images, labels) in enumerate(test_loader):
images = images.cuda(1)
labels = labels.cuda(1)
outputs = model(images)
ret, preds = torch.max(outputs.data, 1)
cnt = preds.eq(labels.data.view_as(preds))
acc = torch.mean(cnt.type(torch.FloatTensor))
val_acc += acc.item() * images.size(0)
if val_acc > best_acc:
print("correct testing samples:", val_acc)
best_acc = val_acc
torch.save(model,
'models/' + 'model_' + str(epoch + 1) + '.pt')
from encode import HOAencode
from DataPreprocessor import array2HOA
sys.path.append('/home/cjf/workspace/201903_dereverLocEnhance/mir_eval_master/')
from mir_eval import separation as sep
bottle = False
DATA_TYPE = 'stft'
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
checkpoint = torch.load('./models/ckpoint_epo30_STFT_speech_bot0_lr0.001_wd0.0001_#res3.tar')
block = ResBlock(256, 256, bottleneck=bottle)
model = ResNet(block, numOfResBlock=3, input_shape=(64, 22, 255), data_type=DATA_TYPE).to(device)
model.load_state_dict(checkpoint['state_dict'])
#print(checkpoint['state_dict'])
def enhance(block, angles):
"""
:param block: given a block of signal
:param angles: directions to be enchanced
return: shifted signals according to the delay
"""
rec_signal = extract_sig(block, angles)
shifted_signals = np.zeros((rec_signal.shape[0] + 2 * c.tf_max_len, len(angles)), dtype=float)
lags = np.zeros(len(angles))
# lags = (np.array([3, 5, 8.544, 8.544, 11]) - 3) / c.speed_of_sound * c.fs # ideal delay
for id, angle in enumerate(angles):
def main(config, resume):
# Dataset
fine_dataset = self_defined_dataset(config)
# Dataloder
train_loader = DataLoader(fine_dataset,
shuffle=True,
batch_size=config['batch_size'],
num_workers=8)
val_loader = DataLoader(fine_dataset,
shuffle=False,
batch_size=config['batch_size'],
num_workers=8)
test_loader = DataLoader(fine_dataset,
shuffle=False,
batch_size=config['batch_size'],
num_workers=8)
# Model
start_epoch = 0
if config['model_name'].startswith('resnet'):
model = ResNet(config)
elif config['model_name'].startswith('densenet'):
model = DenseNet(config)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
#Optimizer
optimizer = torch.optim.Adam(model.parameters(),
lr=config['learning_rate'],
weight_decay=1e-5)
# if use pretrained models
if resume:
filepath = config['pretrain_path']
start_epoch, learning_rate, optimizer = load_ckpt(model, filepath)
start_epoch += 1
# if use multi-GPU
if torch.cuda.device_count() > 1:
print("Let's use", torch.cuda.device_count(), "GPUs!")
model = nn.DataParallel(model)
model.to(device)
#resume or not
if start_epoch == 0:
print("Grand New Training")
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
optimizer, patience=config['switch_learning_rate_interval'])
if not resume:
learning_rate = config['learning_rate']
# training part
if config['if_train']:
for epoch in range(start_epoch + 1,
start_epoch + config['num_epoch'] + 1):
loss_tr = train(
train_loader, model, optimizer, epoch,
config) #if training, delete learning rate and add optimizer
if config['if_valid'] and epoch % config[
'valid_epoch_interval'] == 0:
with torch.no_grad():
loss_val = valid(val_loader, model, epoch, config)
scheduler.step(loss_val)
save_ckpt(model, optimizer, epoch, loss_tr, loss_val, config)
test(test_loader, model, config)
#store_config(config)
print("Training finished ...")
x = deconv_output.cpu().detach().numpy()
x = x.squeeze(0)
x = np.transpose(x, (1,2,0))
x = normalization(x)
x = preprocess_image(x, resize_im=False)
x = recreate_image(x)
if not os.path.exists('./deconv/'+ png_dir):
os.makedirs('./deconv/'+ png_dir)
im_path = './deconv/'+ png_dir+ '/layer_vis_' + str(demode) +'_' + str(index)+ '.jpg'
save_image(x, im_path)
if __name__ == '__main__':
#get model and data
net = ResNet()
net.eval()
net = net.cuda()
net.load_state_dict(torch.load('./cifar_net.pth'))
# print(net.state_dict().keys())
# for name, module in net._modules.items():
# for name, module in module._modules.items():
# print(name)
# for name, param in net.named_parameters():
# print(name)
# net2 = ResNet_deconv(demode=1)
# net2 = net2.cuda()
# encorder = ResNet_encorder(demode=2)
# encorder = encorder.cuda()
# decorder = ResNet_decorder2(demode=2)
# decorder = decorder.cuda()
def main():
"""
"""
exp_path = Path.cwd()
torch.cuda.empty_cache()
# get user to choose training config
config_file = "ResNet.json"
# Load Test Parameters
with open(config_file, "r") as f:
x = json.load(f)
hyperparams = x["HYPERPARAMETERS"]
name = x["model"]
n_bands = hyperparams['n_bands']
patch_size = hyperparams['patch_size']
_, path = utils.experiment_path_build(x)
c_drive_docs = Path(x["LOGGING"]["log_location"])
log_file = Path(x["LOGGING"]["log_file"])
default_device = torch.device(
'cuda:0' if torch.cuda.is_available() else 'cpu')
# Parameters
validation_split = hyperparams["validation_split"]
test_split = hyperparams["test_split"]
random_seed = x["BASIC_PARAMETERS"]["random_seed"]
shuffle_dataset = x["LOCATIONS"]["shuffle_dataset"]
disp_batch = hyperparams["disp_batch"] # images to display on test
epochs = hyperparams["epoch"]
bs = hyperparams["batch_size"]
n_jobs = x["BASIC_PARAMETERS"]["n_jobs"]
lr = hyperparams["learning_rate"]
HSI_ds = HSI_Dataset(path)
_, classes = utils.get_classes(path)
num_classes = len(classes)
train_ds_sampler, valid_ds_sampler, test_ds_sampler = train_test_valid_split(
HSI_ds, shuffle_dataset, validation_split, test_split, random_seed)
phases = make_phases(HSI_ds, train_ds_sampler,
valid_ds_sampler, test_ds_sampler, bs=32, n_jobs=4, disp_batch=disp_batch)
model = Net(ResidualBlock, [2, 4, 8], in_channels=n_bands,
num_classes=num_classes).to(default_device)
# Lee Model Call
# model = Net(n_bands, n_classes, patch_size)
opt = optim.Adam(model.parameters(), lr=1e-2)
training_params = {'Dataset Path': path,
'Experiment Path': exp_path,
'number of classes': num_classes,
'number of bands': n_bands,
'patch size': patch_size,
'test_train Split': {
'validation split': validation_split,
'shuffle dataset': shuffle_dataset,
'random seed': random_seed},
'Batch Size': bs,
"Number of Jobs": n_jobs,
"Learning Rate": lr,
"Scheduler": "One Cycle Schedule"
}
cb = CallbacksGroup([
RollingLoss(),
Accuracy(),
Scheduler(
OneCycleSchedule(t=len(phases[0].loader) * epochs),
mode='batch'
),
StreamLogger(),
CSVLogger(c_drive_docs.joinpath(log_file), training_params),
ProgressBar(),
save_model(c_drive_docs, name),
test_images(path=c_drive_docs, batch_size=5, classes=classes)
])
# save_model(exp_path, name)
train(model, opt, phases, cb, epochs=epochs,
device=default_device, loss_fn=F.cross_entropy)
lr_history = pd.DataFrame(cb['scheduler'].parameter_history('lr'))
ax = lr_history.plot(figsize=(8, 6))
ax.set_xlabel('Training Batch Index')
ax.set_ylabel('Learning Rate')
fig = ax.get_figure()
file_loc = [str(c_drive_docs) + "\\checkpoints\\lr-test.jpg"]
s = ""
s = s.join(file_loc)
conf_path = Path(s)
fig.savefig(conf_path)
transform=transforms.ToTensor(),
download=True)
test_dataset = torchvision.datasets.CIFAR10(root='./',
train=False,
transform=transforms.ToTensor())
train_loader = torch.utils.data.DataLoader(dataset=train_dataset,
batch_size=opt.batch,
shuffle=True)
test_loader = torch.utils.data.DataLoader(dataset=test_dataset,
batch_size=opt.batch,
shuffle=False)
classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse',
'ship', 'truck')
net = ResNet()
# define optimization method
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(net.parameters(), lr=opt.lr, weight_decay=opt.wd)
if opt.is_gpu:
net = net.cuda()
net = torch.nn.DataParallel(net,
device_ids=range(torch.cuda.device_count()))
cudnn.benchmark = True
start_epoch = 0
# train dataset
for epoch in range(start_epoch, opt.epochs + start_epoch):
