def build_model():
global cfg, loaded_ckpt, lg
if rank == 0:
lg.info('==> Building model..')
net: torch.nn.Module = {
'FCNet': FCNet,
}[cfg.model.name](**cfg.model.kwargs)
init_params(net)
if loaded_ckpt is not None:
net.load_state_dict(loaded_ckpt['model'])
num_para = sum(p.numel() for p in net.parameters()) / 1e6
if rank == 0:
lg.info(
f'==> Building model complete, type: {type(net)}, param:{num_para} * 10^6.\n'
)
return net.cuda() if using_gpu else net
def __init__(self, num_classes, backbone, BatchNorm):
super(Decoder, self).__init__()
if backbone in {'resnet50', 'resnet101'} or backbone == 'drn':
low_level_inplanes = 256
elif backbone == 'xception':
low_level_inplanes = 128
elif backbone == 'mobilenet':
low_level_inplanes = 24
else:
raise NotImplementedError
self.conv1 = nn.Conv2d(low_level_inplanes, 48, 1, bias=False)
self.bn1 = BatchNorm(48)
self.relu = nn.ReLU()
self.last_conv = nn.Sequential(
nn.Conv2d(304, 256, kernel_size=3, stride=1, padding=1,
bias=False), BatchNorm(256), nn.ReLU(), nn.Dropout(0.5),
nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1,
bias=False), BatchNorm(256), nn.ReLU(), nn.Dropout(0.1),
nn.Conv2d(256, num_classes, kernel_size=1, stride=1))
init_params(self)
def index():
url = request.args.get("url", "")
if url == "":
vtuber = request.args.get("vtuber", "")
if vtuber != "":
# print(request.args)
save_data(request.args)
params = init_params()
return render_template("index.html", params=params)
else:
# URLが入力された時
_id = url.split("?v=")[-1]
if exists_id(DF, _id):
params = init_params()
params["existsUrl"] = True
params["url"] = url
print(params)
return render_template("index.html", params=params)
data = search_youtube(_id)
# print(data)
params = create_params(data)
return render_template("index.html", params=params)
def init_weights(self, u_emb_para, i_emb_para, rec_para):
"""
Initialize parameters
:param u_emb_para: User embedding parameters
:param i_emb_para: Item embedding parameters
:param rec_para: Recommendation parameters
"""
# Initialize user embedding parameters
init_params(self.user_embedding.parameters(), u_emb_para)
# Initialize item embedding parameters
init_params(self.item_embedding.parameters(), i_emb_para)
# Initialize recommendation model parameters
init_params(self.rec_model.parameters(), rec_para)
def defaultXMLGen_Simulator(sensor_config_file):
print('\n Default Components : Version [%s]' % version)
print('\n Config File %s' % sensor_config_file)
#print('\n Config File %s' %sys.argv[1]
#if (len(sys.argv) < 2 ) :
# utils.error('Usage default XML <input_filename>')
# return
#sensor_config_file=sys.argv[1]
"""
System Parameters Related to Project
"""
sys_params = {}
utils.init_params(sys_params)
#
utils.get_params(sensor_config_file, sys_params)
#
for wdr_mode in ['linear', 'wdr']:
GEN_Params = {}
init_generic_params(GEN_Params, sys_params)
GEN_Params['WDR_MODE'] = wdr_mode
AWB_Params = {}
init_dcc_regions(sys_params, AWB_Params)
generate_awb_xml(sys_params, AWB_Params, GEN_Params)
RGB2RGB1_Params = {}
init_dcc_regions(sys_params, RGB2RGB1_Params)
generate_rgb2rgb1_xml(sys_params, RGB2RGB1_Params, GEN_Params)
CFAI_Params = {}
init_dcc_regions(sys_params, CFAI_Params)
generate_cfai_xml(sys_params, CFAI_Params, GEN_Params)
H3A_AEWB_Params = {}
init_dcc_regions(sys_params, H3A_AEWB_Params)
generate_h3a_aewb_xml(sys_params, H3A_AEWB_Params, GEN_Params)
LDC_Params = {}
init_dcc_regions(sys_params, LDC_Params)
generate_ldc_xml(sys_params, LDC_Params, GEN_Params)
viss_blc_Params = {}
init_dcc_regions(sys_params, viss_blc_Params)
generate_viss_blc_xml(sys_params, viss_blc_Params, GEN_Params)
NSF4_Params = {}
init_dcc_regions(sys_params, NSF4_Params)
generate_nsf4_xml(sys_params, NSF4_Params, GEN_Params)
H3AMUX_Params = {}
init_dcc_regions(sys_params, H3AMUX_Params)
generate_h3amux_xml(sys_params, H3AMUX_Params, GEN_Params)
if (wdr_mode == 'wdr'):
GLBCE_Params = {}
init_dcc_regions(sys_params, GLBCE_Params)
generate_glbce_xml(sys_params, GLBCE_Params, GEN_Params)
DECMP_Params = {}
init_dcc_regions(sys_params, DECMP_Params)
generate_decmp_xml(sys_params, DECMP_Params, GEN_Params)
generate_dcc_gen_script(sys_params, GEN_Params)
return
netD.apply(weights_init)
if opt.netD != '':
netD.load_state_dict(torch.load(opt.netD))
logger.info(netD)
##### Classifier #####
if opt.netT != '':
chk = torch.load(opt.netT)
netT = chk['netT']
best_acc = chk['acc']
netT_epoch = chk['epoch']
else:
# netT = ResNet18()
netT = MnistClassifier(source_channels, target_channels, num_classes,
opt.ngpu)
init_params(netT)
best_acc = 0
netT_epoch = 0
logger.info(netT)
criterion_D = nn.BCEWithLogitsLoss()
criterion_G = nn.BCEWithLogitsLoss()
criterion_T = nn.CrossEntropyLoss()
inputs = torch.FloatTensor(opt.batchSize, 3, opt.imageSize, opt.imageSize)
noise = torch.FloatTensor(opt.batchSize, nz, 1, 1)
fixed_noise = torch.FloatTensor(opt.batchSize, nz).uniform_(-1, 1)
label = torch.FloatTensor(opt.batchSize)
real_label = 1
fake_label = 0
def run(args):
if not args.single_test:
import pidfile
resfile = pidfile.exclusive_dirfn(
os.path.join(args.r_dir, args.save_dir))
if args.log_per_task:
args.prec_log = args.iters
args.loss_log = args.iters
# -create plots- and results-directories if needed
if not os.path.isdir(args.r_dir):
os.mkdir(args.r_dir)
if args.pdf and not os.path.isdir(args.p_dir):
os.mkdir(args.p_dir)
# set cuda
cuda = torch.cuda.is_available() and args.cuda
device = torch.device("cuda" if cuda else "cpu")
# set random seeds
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if cuda:
torch.cuda.manual_seed(args.seed)
torch.cuda.manual_seed_all(args.seed)
scenario = args.scenario
#-------------------------------------------------------------------------------------------------
# DATA
#-------------------------------------------------------------------------------------------------
(train_datasets, test_datasets), config = get_multitask_experiment(
args,
name=args.experiment,
scenario=scenario,
tasks=args.tasks,
data_dir=args.d_dir,
verbose=True,
exception=True if args.seed == 0 else False,
)
args.tasks = len(config['labels_per_task'])
args.labels_per_task = config['labels_per_task']
if not args.task_boundary:
args.iterations_per_virtual_epc = config['iterations_per_virtual_epc']
args.task_dict = config['task_dict']
#-------------------------------------------------------------------------------------------------
# MODEL
#-------------------------------------------------------------------------------------------------
if args.ebm:
model = EBM(args,
image_size=config['size'],
image_channels=config['channels'],
classes=config['num_classes'],
fc_units=args.fc_units).to(device)
else:
model = Classifier(args,
image_size=config['size'],
image_channels=config['channels'],
classes=config['num_classes'],
fc_units=args.fc_units).to(device)
if args.experiment == 'cifar100':
model = utils.init_params(model, args)
for param in model.convE.parameters():
param.requires_grad = False
if args.pretrain:
checkpoint = torch.load(args.pretrain)
best_acc = checkpoint['best_acc']
checkpoint_state = checkpoint['state_dict']
print(
'-----------------------------------------------------------------------------'
)
print('load pretrained model %s' % args.pretrain)
print('best_acc', best_acc)
print(
'-----------------------------------------------------------------------------'
)
model_dict = model.fcE.state_dict()
checkpoint_state = {
k[7:]: v
for k, v in checkpoint_state.items() if k[7:] in model_dict
} ## remove module.
del checkpoint_state['classifier.weight']
del checkpoint_state['classifier.bias']
if 'y_ebm.weight' in checkpoint_state:
del checkpoint_state['y_ebm.weight']
model_dict.update(checkpoint_state)
model.fcE.load_state_dict(model_dict)
for param in model.fcE.model.parameters():
param.requires_grad = False
model.optim_list = [{
'params':
filter(lambda p: p.requires_grad, model.parameters()),
'lr':
args.lr
}]
model.optim_type = args.optimizer
if model.optim_type in ("adam", "adam_reset"):
model.optimizer = optim.Adam(model.optim_list, betas=(0.9, 0.999))
elif model.optim_type == "sgd":
model.optimizer = optim.SGD(model.optim_list)
else:
raise ValueError(
"Unrecognized optimizer, '{}' is not currently a valid option".
format(args.optimizer))
#-------------------------------------------------------------------------------------------------
# CL-STRATEGY: ALLOCATION
#-------------------------------------------------------------------------------------------------
# Elastic Weight Consolidation (EWC)
if isinstance(model, ContinualLearner):
model.ewc_lambda = args.ewc_lambda if args.ewc else 0
if args.ewc:
model.fisher_n = args.fisher_n
model.gamma = args.gamma
model.online = args.online
model.emp_FI = args.emp_fi
# Synpatic Intelligence (SI)
if isinstance(model, ContinualLearner):
model.si_c = args.si_c if args.si else 0
if args.si:
model.epsilon = args.epsilon
#-------------------------------------------------------------------------------------------------
# Get parameter-stamp (and print on screen)
#-------------------------------------------------------------------------------------------------
param_stamp = get_param_stamp(args, model.name, verbose=True)
param_stamp = param_stamp + '--' + args.model_name
# -define [precision_dict] to keep track of performance during training for storing and for later plotting in pdf
precision_dict = evaluate.initiate_precision_dict(args.tasks)
#-------------------------------------------------------------------------------------------------#
#---------------------#
#----- CALLBACKS -----#
#---------------------#
solver_loss_cbs = [
cb._solver_loss_cb(log=args.loss_log,
model=model,
tasks=args.tasks,
iters_per_task=args.iters)
]
eval_cb = cb._eval_cb(log=args.prec_log,
test_datasets=test_datasets,
visdom=args.visdom,
precision_dict=None,
iters_per_task=args.iters,
test_size=args.prec_n,
labels_per_task=config['labels_per_task'],
scenario=scenario)
eval_cb_full = cb._eval_cb(log=args.iters,
test_datasets=test_datasets,
precision_dict=precision_dict,
iters_per_task=args.iters,
labels_per_task=config['labels_per_task'],
scenario=scenario)
eval_cbs = [eval_cb, eval_cb_full]
#-------------------------------------------------------------------------------------------------
# TRAINING
#-------------------------------------------------------------------------------------------------
print("--> Training:")
start = time.time()
if args.task_boundary:
train_cl(args,
model,
train_datasets,
scenario=scenario,
labels_per_task=config['labels_per_task'],
iters=args.iters,
batch_size=args.batch,
eval_cbs=eval_cbs,
loss_cbs=solver_loss_cbs)
else:
train_cl_noboundary(args,
model,
train_datasets,
scenario=scenario,
labels_per_task=config['labels_per_task'],
iters=args.iters,
batch_size=args.batch,
eval_cbs=eval_cbs,
loss_cbs=solver_loss_cbs)
training_time = time.time() - start
#-------------------------------------------------------------------------------------------------
# EVALUATION
#-------------------------------------------------------------------------------------------------
print("\n\n--> Evaluation ({}-incremental learning scenario):".format(
args.scenario))
if args.ebm:
precs = [
evaluate.validate_ebm(args,
model,
test_datasets[i],
verbose=False,
test_size=None,
task=i + 1,
with_exemplars=False,
current_task=args.tasks)
for i in range(args.tasks)
]
else:
precs = [
evaluate.validate(args,
model,
test_datasets[i],
verbose=False,
test_size=None,
task=i + 1,
with_exemplars=False,
current_task=args.tasks)
for i in range(args.tasks)
]
print("\n Precision on test-set (softmax classification):")
for i in range(args.tasks):
print(" - Task {}: {:.4f}".format(i + 1, precs[i]))
average_precs = sum(precs) / args.tasks
print('average precision over all {} tasks: {:.4f}'.format(
args.tasks, average_precs))
#-------------------------------------------------------------------------------------------------
# OUTPUT
#-------------------------------------------------------------------------------------------------
if not os.path.exists(os.path.join(args.r_dir, args.save_dir)):
os.makedirs(os.path.join(args.r_dir, args.save_dir))
output_file = open(
"{}/{}/{}.txt".format(args.r_dir, args.save_dir, param_stamp), 'w')
output_file.write("Training time {} \n".format(training_time))
for i in range(args.tasks):
output_file.write(" - Task {}: {:.4f}".format(i + 1, precs[i]))
output_file.write("\n")
output_file.write(' - Average {}\n'.format(average_precs))
output_file.close()
file_name = "{}/{}/{}".format(args.r_dir, args.save_dir, param_stamp)
utils.save_object(precision_dict, file_name)
if args.pdf:
pp = visual_plt.open_pdf("{}/{}/{}.pdf".format(args.r_dir,
args.save_dir,
param_stamp))
# -show metrics reflecting progression during training
figure_list = [] #-> create list to store all figures to be plotted
# -generate all figures (and store them in [figure_list])
figure = visual_plt.plot_lines(
precision_dict["all_tasks"],
x_axes=precision_dict["x_task"],
line_names=['task {}'.format(i + 1) for i in range(args.tasks)])
figure_list.append(figure)
figure = visual_plt.plot_lines([precision_dict["average"]],
x_axes=precision_dict["x_task"],
line_names=['average all tasks so far'])
figure_list.append(figure)
# -add figures to pdf (and close this pdf).
for figure in figure_list:
pp.savefig(figure)
pp.close()
if not args.single_test:
resfile.done()
decode_original, result, speed, 100 * packet_loss)
tkinter.messagebox.showinfo('传输结果', show_text)
def init_ui(self):
'''
描述：初始化gui
参数：无
返回：无
'''
self.window = Tk()
self.label1 = Label(self.window, text="未开始录音")
self.label1.grid(row=0, column=0, stick=W, pady=10)
self.label2 = Label(self.window, text="原先的信息（必须是ASCII码）")
self.label2.grid(row=1, column=0, stick=W, pady=10)
self.entry1 = Entry(self.window, width=100)
self.entry1.grid(row=1, column=1, stick=W, pady=10)
self.button1 = Button(self.window,
text='开始录音',
command=self.start_record)
self.button1.grid(row=2, column=0, stick=W, pady=10)
self.button2 = Button(self.window,
text='结束录音',
command=self.stop_record)
self.button2.grid(row=2, column=1, stick=W, pady=10)
self.window.mainloop()
if __name__ == "__main__":
args = init_params()
receiver = Receiver(args)
receiver.init_ui()
# 0. Postpone training for TRAINING_DELAY seconds.
time.sleep(TRAINING_DELAY)
# 1. Create a directory for resume (model) parameters, CSV logs, and tensor-board summary logs.
log_dir = utils.mkdir_for_logs(MODEL.__name__, DATASET_FOLDER_NAME,
OBJ_FUNC)
# 2. Instantiate a tensorboard writer and CSV writers.
tensor_board_writer = SummaryWriter(log_dir=log_dir)
train_csv_file, train_csv_writer, valid_csv_file, valid_csv_writer = \
utils.csv_train_valid_writer_for_logs(log_dir=log_dir)
# 3. Instantiate and initialize a CNN model.
model = MODEL(num_classes=NUM_CLASSES)
utils.init_params(model)
# 4. Load the model to CPU or GPU.
device = 'cuda' if torch.cuda.is_available() else 'cpu'
model = model.to(device)
# 5. (If GPU,) Turn on 'Data Parallel' mode.
if device == 'cuda':
model = torch.nn.DataParallel(model)
cudnn.benchmark = True
print('model_info:', MODEL.__name__, '/ device:', device, '/ obj.:',
OBJ_FUNC, '/ dataset:', DATASET_FOLDER_NAME)
# 6. Load the train and valid dataset.
train_loader, valid_loader = \
def main(
arch = 'resnet_weight',
num_epochs = 160,
batch_size = 256,
load_checkpoint_dir = 'resnet56/2/model_best.pth.tar',
save_checkpoint_dir = 'resnet56/2/distill/scratch/',
save_prune = 'resnet56/2/',
parrallel=True,
scratch=True):
# Data loading code
transform = transforms.Compose([
transforms.ToTensor(),
])
# train_mean, train_std = get_mean_and_std(torchvision.datasets.CIFAR10('./cifar10_data', train=True, download=True, transform=transform))
# test_mean, test_std = get_mean_and_std(torchvision.datasets.CIFAR10('./cifar10_data', train=False, download=True, transform=transform))
# print(train_mean, train_std, test_mean, test_std)
data_transform = {
'train' : transforms.Compose([
transforms.Pad(4),
transforms.RandomCrop(32),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
# transforms.Normalize(train_mean, train_std),
transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
]),
'test' : transforms.Compose([
transforms.ToTensor(),
# transforms.Normalize(test_mean, test_std),
transforms.Normalize((0.4942, 0.4851, 0.4504), (0.2020, 0.1991, 0.2011)),
]) }
train_dataset = torchvision.datasets.CIFAR10('./cifar10_data', train=True, transform=data_transform['train'], download=False)
test_dataset = torchvision.datasets.CIFAR10('./cifar10_data', train=False, transform=data_transform['test'], download=False)
train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False)
# Model loading code
# Originmodel = OriginNetwork(models.resnet50(pretrained=False, num_classes=10), num_classes=10)
# Submodel = SubNetwork(models.resnet50(pretrained=False, num_classes=10), num_classes=10)
# Originmodel = models.__dict__[arch](dataset='cifar10', depth=56) # Filter pruning
Originmodel = models.__dict__[arch](num_classes=10, depth=56) # Weight pruning
# Submodel = models.__dict__[arch](dataset='cifar10', depth=56)
# Selecting device
device = torch.device('cuda:1' if torch.cuda.is_available() else 'cpu')
torch.cuda.set_device(device)
Originmodel.to(device)
# Submodel.to(device) # resnet56_pruning 에서 device로 옮겨짐.
# Loading Checkpoint
load_checkpoint(Originmodel, checkpoint_dir=load_checkpoint_dir, parrallel=parrallel)
unpruned_acc = test_solo(test_loader, Originmodel, device)
# Submodel, num_parameters = resnet56_pruning(Originmodel, dataset='cifar10', save=save_prune, device=device, v='A') # Filter
Submodel = resnet56_weight_pruning(Originmodel, percent=0.3) # Weight
pruned_acc = test_solo(test_loader, Submodel, device)
# Pruned before & after accuracy save
with open(os.path.join(save_prune, "prune.txt"), "w") as fp:
fp.write("Before pruning Test accuracy : \n"+str(unpruned_acc.item())+"\n")
# fp.write("Number of parameters: \n"+str(num_parameters)+"\n")
fp.write("Test accuracy: \n"+str(pruned_acc.item())+"\n")
# Scratch training
if scratch:
Submodel.train()
init_params(Submodel)
test_solo(test_loader, Submodel, device)
# Loss & Optimizer
criterion = distill_loss(origin_model=Originmodel, subnet_model=Submodel, temperature=5)
optimizer = torch.optim.SGD(Submodel.parameters(), lr=0.1, momentum=0.9, weight_decay=1e-4)
# scheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones=[15,25,30], gamma= 0.1)
scheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones=[80,120], gamma= 0.1) # Scratch
# Visualization
writer = SummaryWriter(os.path.join(save_checkpoint_dir, 'logs'))
# Logging Metrics
title = 'ResNet56'
logger = Logger(os.path.join(save_checkpoint_dir, 'log.txt'), title=title)
logger.set_names(['Learning Rate', 'Train Loss', 'Test Loss', 'Train Acc', 'Test Acc'])
best_prec1 = 0
for epoch in range(num_epochs):
print('')
# Train : Distilling knowledge to Subnet from origin
train_loss, train_acc = train(train_loader, Originmodel, Submodel, criterion, optimizer, epoch, device)
# Test
test_loss, test_acc = test(test_loader, Originmodel, Submodel, criterion, epoch, device)
lr = optimizer.param_groups[0]['lr']
# append logger file
logger.append([lr, train_loss, test_loss, train_acc, test_acc])
# visualize training process using tensorboardx
writer.add_scalar('learning rate', lr)
writer.add_scalars('loss', {'train loss':train_loss, 'test loss':test_loss}, epoch+1)
writer.add_scalars('accuracy', {'train accuracy':train_acc, 'test accuracy':test_acc}, epoch+1)
# Learning Rate scheduler
scheduler.step()
is_best = test_acc > best_prec1
best_prec1 = max(test_acc, best_prec1)
save_checkpoint(state={
'epoch':epoch+1,
'arch':arch,
'state_dict':Submodel.state_dict(),
'best_prec1':best_prec1,
'optimizer':optimizer.state_dict(),
'loss':criterion.state_dict()
}, is_best=is_best, checkpoint=save_checkpoint_dir)
logger.close()
logger.plot()
savefig(os.path.join(save_checkpoint_dir, 'log.jpg'))
writer.close()
print(f'Best Accuracy : {best_prec1}')
device = 'cuda' if torch.cuda.is_available() else 'cpu'
net = net.to(device)
if device == 'cuda':
net = torch.nn.DataParallel(net)
cudnn.benchmark = True
if args.resume:
# Load checkpoint.
print('==> Resuming from checkpoint..')
assert os.path.isdir('checkpoint'), 'Error: no checkpoint directory found!'
checkpoint = torch.load('./checkpoint/ckpt.t7')
net.load_state_dict(checkpoint['net'])
best_acc = checkpoint['acc']
start_epoch = checkpoint['epoch']
else:
init_params(net)
total_params = sum(p.numel() for p in net.parameters() if p.requires_grad)
print('Number of trainable parameters {}'.format(total_params))
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(net.parameters(),
lr=args.lr,
momentum=0.9,
weight_decay=5e-4)
scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min')
# Training
def train(epoch):
print('\nEpoch: %d' % epoch)
def __init__(self,
model_name,
block,
layers,
output_stride,
BatchNorm,
pretrained=True,
enable_dff=False,
pretrained_path=None):
self.model_name = model_name
self.inplanes = 64
super(ResNet, self).__init__()
self.enable_dff = enable_dff
blocks = [1, 2, 4] if (
not enable_dff) else [1, 1, 1] # rloss: [1,2,4]; dff: [1,1,1]
if output_stride == 16:
strides = [1, 2, 2, 1]
dilations = [1, 1, 1, 2] if (not enable_dff) else [
2, 2, 2, 4
] # rloss: [1,1,1,2]; dff: [2,2,2,4] Zhiwei
elif output_stride == 8:
strides = [1, 2, 1, 1]
dilations = [1, 1, 2, 4]
else:
raise NotImplementedError
# Modules Zhiwei
if enable_dff:
self.conv1 = nn.Conv2d(3,
64,
kernel_size=7,
stride=1,
padding=3,
bias=False)
else:
self.conv1 = nn.Conv2d(3,
64,
kernel_size=7,
stride=2,
padding=3,
bias=False)
self.bn1 = BatchNorm(64)
self.relu = nn.ReLU(inplace=True)
self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
self.pretrained_path = pretrained_path
self.layer1 = self._make_layer(block,
64,
layers[0],
stride=strides[0],
dilation=dilations[0],
BatchNorm=BatchNorm)
self.layer2 = self._make_layer(block,
128,
layers[1],
stride=strides[1],
dilation=dilations[1],
BatchNorm=BatchNorm)
self.layer3 = self._make_layer(block,
256,
layers[2],
stride=strides[2],
dilation=dilations[2],
BatchNorm=BatchNorm)
self.layer4 = self._make_MG_unit(block,
512,
blocks=blocks,
stride=strides[3],
dilation=dilations[3],
BatchNorm=BatchNorm)
# self.layer4 = self._make_layer(block, 512, layers[3], stride=strides[3], dilation=dilations[3], BatchNorm=BatchNorm)
init_params(self)
# self._init_weight()
if pretrained:
self._load_pretrained_model()
def train():
if prm.optimizer.lower() == 'adam':
optimizer = adam
elif prm.optimizer.lower() == 'sgd':
optimizer = sgd
elif prm.optimizer.lower() == 'rmsprop':
optimizer = rmsprop
elif prm.optimizer.lower() == 'adadelta':
optimizer = adadelta
options = locals().copy()
print 'parameters:', str(options)
prm_k = vars(prm).keys()
prm_d = vars(prm)
prm_k.sort()
for x in prm_k:
if not x.startswith('__'):
print x, '=', prm_d[x]
print 'loading Vocabulary...'
vocab = utils.load_vocab(prm.vocab_path, prm.n_words)
options['vocab'] = vocab
options['vocabinv'] = {}
for k, v in vocab.items():
options['vocabinv'][v] = k
print options
print 'Loading Environment...'
if prm.engine.lower() == 'lucene':
import lucene_search
options['engine'] = lucene_search.LuceneSearch()
elif prm.engine.lower() == 'elastic':
import elastic_search
options['engine'] = elastic_search.ElasticSearch()
print 'Loading Dataset...'
dh5 = dataset_hdf5.DatasetHDF5(prm.dataset_path)
qi_train = dh5.get_queries(dset='train')
dt_train = dh5.get_doc_ids(dset='train')
qi_valid = dh5.get_queries(dset='valid')
dt_valid = dh5.get_doc_ids(dset='valid')
qi_test = dh5.get_queries(dset='test')
dt_test = dh5.get_doc_ids(dset='test')
if prm.train_size == -1:
train_size = len(qi_train)
else:
train_size = min(prm.train_size, len(qi_train))
if prm.valid_size == -1:
valid_size = len(qi_valid)
else:
valid_size = min(prm.valid_size, len(qi_valid))
if prm.test_size == -1:
test_size = len(qi_test)
else:
test_size = min(prm.test_size, len(qi_test))
print '%d train examples' % len(qi_train)
print '%d valid examples' % len(qi_valid)
print '%d test examples' % len(qi_test)
# This create the initial parameters as np ndarrays.
# Dict name (string) -> np ndarray
params, exclude_params = utils.init_params(options)
if prm.wordemb_path:
print 'loading pre-trained word embeddings'
params = utils.load_wemb(params, vocab)
options['W'] = params['W']
if prm.reload_model:
utils.load_params(prm.reload_model, params)
print 'Building model'
# This create Theano Shared Variable from the parameters.
# Dict name (string) -> Theano Tensor Shared Variable
# params and tparams have different copy of the weights.
tparams = utils.init_tparams(params)
for kk, value in tparams.iteritems():
tparams[kk] = theano.shared(value, name=kk)
iin, out, updates, f_pred, consider_constant \
= build_model(tparams, options)
# get only parameters that are not in the exclude_params list
tparams_ = OrderedDict([(kk, vv) for kk, vv in tparams.iteritems()
if kk not in exclude_params])
grads = tensor.grad(out[0],
wrt=utils.itemlist(tparams_),
consider_constant=consider_constant)
lr = tensor.scalar(name='lr')
f_grad_shared, f_update = optimizer(lr, tparams_, grads, iin, out, updates)
history_errs = []
best_p = None
if prm.validFreq == -1:
validFreq = len(qi_train) / prm.batch_size_train
else:
validFreq = prm.validFreq
if prm.saveFreq == -1:
saveFreq = len(qi_train) / prm.batch_size_train
else:
saveFreq = prm.saveFreq
uidx = 0 # the number of update done
estop = False # early stop
start_time = time.time()
print 'Optimization'
try:
for eidx in xrange(prm.max_epochs):
n_samples = 0
# Get new shuffled index for the training set.
kf = utils.get_minibatches_idx(len(qi_train),
prm.batch_size_train,
shuffle=True)
for _, train_index in kf:
st = time.time()
uidx += 1
qi, qi_i, qi_lst, D_gt_id, D_gt_url = get_samples(
qi_train, dt_train, train_index, options)
# share the current queries with the search engine.
options['current_queries'] = qi_lst
n_samples += len(qi)
is_train = 1.
out = f_grad_shared(qi_i, D_gt_id, is_train)
cost = out.pop(0)
cost_ent = out.pop(0)
lr_t = f_update(prm.lrate)
if np.isnan(cost) or np.isinf(cost):
print 'NaN detected'
return 1., 1., 1.
print "options['reformulated_queries']", options[
'reformulated_queries']
if np.mod(uidx, prm.dispFreq) == 0:
print '\n================================================================================'
print 'Epoch', eidx, 'Update', uidx, 'Cost', cost, 'LR_t', lr_t
print 'Time Minibatch Update: ' + str(time.time() - st)
print 'Input Query: ', qi[0].replace('\n', '\\n')
print
print 'Target Docs: ', str(D_gt_url[0])
print
print 'Input Query Vocab: ', utils.idx2text(
qi_i[0], options['vocabinv'])
for ii in range(prm.n_iterations):
prob = out.pop(0)
ans = out.pop(0)
metrics = out.pop(0)
bl = out.pop(0)
cost_bl = out.pop(0)
D_id = out.pop(0)
print "prob", prob
print "ans", ans
print "bl", bl
print "cost_bl", cost_bl
print "D_id", D_id
print("current_queries",
len(options['current_queries']),
options['current_queries'])
print
print 'Iteration', ii
print 'Baseline Value', bl.mean(), 'Cost', cost_bl
print ' '.join(prm.metrics_map.keys())
print metrics.mean(0)
print
i = 7
print 'Retrieved Docs: ', str([
options['engine'].id_title_map[d_id]
for d_id in D_id[i]
])
print
print 'Reformulated Query:', options[
'reformulated_queries'][ii][i]
print 'Current queries:', options['current_queries'][i]
print
print 'Query ANS: ',
for kk, word in enumerate(
options['current_queries'][i][:ans.shape[1]]):
print "kk, word", kk, word
if word not in options['vocab'] and word != '':
word += '<unk>'
if ans[0, kk] == 1:
word = word.upper()
print str(word),
print
print
print 'prob[:,:,0].max(1).mean(), prob[:,:,0].mean(), prob[:,:,0].min(1).mean()', prob[:, :, 0].max(
1).mean(), prob[:, :,
0].mean(), prob[:, :,
0].min(1).mean()
print 'prob[:,:,1].max(1).mean(), prob[:,:,1].mean(), prob[:,:,1].min(1).mean()', prob[:, :, 1].max(
1).mean(), prob[:, :,
1].mean(), prob[:, :,
1].min(1).mean()
print '==================================================================================\n'
if np.mod(uidx, validFreq) == 0 or uidx == 1:
kf_train = utils.get_minibatches_idx(
len(qi_train),
prm.batch_size_pred,
shuffle=True,
max_samples=train_size)
kf_valid = utils.get_minibatches_idx(
len(qi_valid),
prm.batch_size_pred,
shuffle=True,
max_samples=valid_size)
kf_test = utils.get_minibatches_idx(len(qi_test),
prm.batch_size_pred,
shuffle=True,
max_samples=test_size)
print '\nEvaluating - Training Set'
train_metrics = pred_error(f_pred, qi_train, dt_train,
options, kf_train)
exit()
print '\nEvaluating - Validation Set'
valid_metrics = pred_error(f_pred, qi_valid, dt_valid,
options, kf_valid)
print '\nEvaluating - Test Set'
test_metrics = pred_error(f_pred, qi_test, dt_test,
options, kf_test)
his = [train_metrics, valid_metrics, test_metrics]
history_errs.append(his)
metric_idx = prm.metrics_map[prm.reward.upper()]
if (uidx == 0 or valid_metrics[-1, metric_idx] >=
np.array(history_errs)[:, 1, -1,
metric_idx].max()):
best_p = utils.unzip(tparams)
bad_counter = 0
print '====================================================================================================='
print ' '.join(prm.metrics_map.keys())
print
print 'Train:'
print train_metrics
print
print 'Valid:'
print valid_metrics
print
print 'Test:'
print test_metrics
print
print '====================================================================================================='
if (len(history_errs) > prm.patience
and valid_metrics[-1, metric_idx] <=
np.array(history_errs)[:-prm.patience, 1, -1,
metric_idx].max()):
bad_counter += 1
if bad_counter > prm.patience:
print 'Early Stop!'
estop = True
break
if prm.saveto and np.mod(uidx, saveFreq) == 0:
print 'Saving...',
if best_p is not None:
params = best_p
else:
params = utils.unzip(tparams)
np.savez(prm.saveto, history_errs=history_errs, **params)
print 'Done'
print 'Seen %d samples' % n_samples
if estop:
break
except KeyboardInterrupt:
print "Training interupted"
return
def init_u_mem_weights(self, u_emb_para, mu, tao, i_emb_para, rec_para):
init_u_mem_params(self.user_embedding.parameters(), u_emb_para, mu,
tao)
init_params(self.item_embedding.parameters(), i_emb_para)
init_params(self.rec_model.parameters(), rec_para)
