def test_contrib_sign_ste():
in_data = nd.uniform(-10, 10, shape=30) # 10 and 30 are arbitrary numbers
w_init = float(nd.uniform(-10, 10, shape=1).asscalar())
check_ste(net_type_str="SignSTENET", w_init=w_init, hybridize=True, in_data=in_data)
check_ste(net_type_str="SignSTENET", w_init=w_init, hybridize=False, in_data=in_data)
# Test 0
in_data = nd.array([0]*30) # 10 and 30 are arbitrary numbers
w_init = 0.
check_ste(net_type_str="SignSTENET", w_init=w_init, hybridize=True, in_data=in_data)
check_ste(net_type_str="SignSTENET", w_init=w_init, hybridize=False, in_data=in_data)
def test_Conv2D(use_bias, groups):
x = nd.uniform(shape=(2, 2, 5, 5))
my_conv = MyConv(10,
3,
1,
1,
in_channels=2,
groups=groups,
use_bias=use_bias)
my_conv.initialize()
ref_conv = Conv2D(10,
3,
1,
1,
in_channels=2,
groups=groups,
use_bias=use_bias,
bias_initializer=init.Constant(my_conv.bias.data())
if use_bias else 'zero',
weight_initializer=init.Constant(my_conv.weight.data()))
ref_conv.initialize()
return (my_conv(x) - ref_conv(x)).abs().sum().asscalar()
def operator():
"""
mxnet的ndarray加减乘除
:return:
"""
a = nd.arange(12).reshape(3, 4)
b = nd.uniform(0, 10, shape=(3, 4))
logger.info("ndarray a:")
logger.info(a)
logger.info("ndarray b:")
logger.info(b)
logger.info("ndarray a.exp():")
logger.info(a.exp())
logger.info("ndarray a.sum():")
logger.info(a.sum())
logger.info("ndarray a.max():")
logger.info(a.max())
logger.info("ndarray a.min():")
logger.info(a.min())
logger.info("ndarray a.abs():")
logger.info(a.abs())
logger.info("ndarray a.norm().asscalar():")
logger.info(a.norm().asscalar())
nd_add = a + b
logger.info("ndarray a+b:")
logger.info(nd_add)
nd_sub = a - b
logger.info("ndarray a-b:")
logger.info(nd_sub)
nd_mul = a * b
logger.info("ndarray a*b:")
logger.info(nd_mul)
nd_dev = a / b
logger.info("ndarray a/b:")
logger.info(nd_dev)
nd_dot = nd.dot(a, a.T)
logger.info("ndarray nd.dot(a,a.T):")
logger.info(nd_dot)
# logger.info("ndarray nd.batch_dot(a,a.T):")
# logger.info(
# nd.batch_dot(nd.arange(24).reshape(2, 3, 4), nd.arange(24).reshape(2, 3, 4), nd.arange(24).reshape(2, 3, 4)))
logger.info("ndarray nd.concat(a,b,dim=0)")
logger.info(nd.concat(a, b, dim=0))
logger.info("ndarray nd.concat(a,b,dim=1)")
logger.info(nd.concat(a, b, dim=1))
def test_contrib_round_ste():
# Test with random data
in_data = nd.uniform(-10, 10, shape=30) # 10 and 30 are arbitrary numbers
w_init = float(nd.uniform(-10, 10, shape=1).asscalar())
check_ste(net_type_str="RoundSTENET", w_init=w_init, hybridize=True, in_data=in_data)
check_ste(net_type_str="RoundSTENET", w_init=w_init, hybridize=False, in_data=in_data)
# Test 1.5 (verifies that .5 rounds the same as in round)
in_data = nd.array([1.5]*30) # 10 and 30 are arbitrary numbers
w_init = 1.
check_ste(net_type_str="RoundSTENET", w_init=w_init, hybridize=True, in_data=in_data)
check_ste(net_type_str="RoundSTENET", w_init=w_init, hybridize=False, in_data=in_data)
# Test 0
in_data = nd.array([0]*30) # 10 and 30 are arbitrary numbers
w_init = 0.
check_ste(net_type_str="RoundSTENET", w_init=w_init, hybridize=True, in_data=in_data)
check_ste(net_type_str="RoundSTENET", w_init=w_init, hybridize=False, in_data=in_data)
def argmax():
a = nd.uniform(0, 5, dtype=np.float32, shape=(3, 4))
logger.info("a:")
logger.info(a)
logger.info(nd.ones(shape=(a.shape[0], 1)) * 3)
logger.info(a.argmax(axis=1))
logger.info("a.argmax(axis=1)>3")
b = a.argmax(axis=1) >= nd.ones(shape=(1, a.shape[0])) * 3
logger.info(b)
def test_check_output():
print("<<TEST: Check whether output is right>>")
inputs = nd.uniform(shape=(1, 3, 224, 224))
ref_net = mobilenet1_0(pretrained=True)
my_net = mobilenet1_0(pretrained=True)
merge_bn(my_net)
ref_output = ref_net(inputs)
my_output = my_net(inputs)
print(((my_output - ref_output).abs() /
ref_output.abs()).reshape(-1).sort()[-20:])
print()
def random():
"""
随机生成多维数组
:return:
"""
# a = nd.random_randint(low=0, high=5, shape=(2, 2, 2))
a = nd.uniform(low=0, high=5, shape=(2, 2, 2))
logger.info(a)
b = nd.random_normal(0, 1, (3, 4))
logger.info(b)
# 从指数分布中随机抽取样本
c = nd.random_exponential(lam=2, shape=(2, 2))
logger.info(c)
def load_model(self):
net = DarkNet(input_dim=self.input_dim,
num_classes=self.num_classes) # 基础网络DarkNet
net.initialize(ctx=self.ctx)
if self.params_path.endswith(".params"): # 加载模型
net.load_params(self.params_path)
elif self.params_path.endswith(".weights"):
tmp_batch = nd.uniform(shape=(1, 3, self.input_dim,
self.input_dim),
ctx=self.ctx)
net(tmp_batch)
net.load_weights(self.params_path, fine_tune=False)
else:
print("params {} load error!".format(self.params_path))
exit()
print_info("加载参数: {}".format(self.params_path))
net.hybridize()
return net
def __load_model(self):
"""
加载网络模型
:return: 网络
"""
net = DarkNet(input_dim=self.input_dim, num_classes=self.num_classes) # 基础网络 DarkNet
net.initialize(ctx=self.ctx) # 网络环境 cpu or gpu
print_info("模型: {}".format(self.params_path))
if self.params_path.endswith(".params"): # 加载参数
net.load_params(self.params_path)
elif self.params_path.endswith(".weights"): # 加载模型
tmp_batch = nd.uniform(shape=(1, 3, self.input_dim, self.input_dim), ctx=self.ctx)
net(tmp_batch)
net.load_weights(self.params_path, fine_tune=False)
else:
raise Exception('模型错误') # 抛出异常
net.hybridize() # 编译和优化网络
return net
def test_quantized_Conv2D(use_bias, groups):
x = nd.uniform(shape=(2, 2, 5, 5))
my_conv = MyConv(10,
3,
1,
1,
in_channels=2,
groups=groups,
use_bias=use_bias,
input_dtype='uint8',
weight_dtype='int8',
quantized=True)
my_conv.initialize()
my_res = my_conv(x)
ref_conv = Conv2D(10,
3,
1,
1,
in_channels=2,
groups=groups,
use_bias=use_bias,
bias_initializer=init.Constant(my_conv.bias.data())
if use_bias else 'zero',
weight_initializer=init.Constant(my_conv.weight.data()))
ref_conv.initialize()
ref_res = ref_conv(x)
sim_conv = ref_conv
convert_conv2d = gen_conv2d_converter()
convert_conv2d(sim_conv)
qparams_init((sim_conv))
sim_res = sim_conv(x)
return (((my_res - ref_res).abs() /
ref_res.abs()).reshape(-1).sort()[-20:],
((my_res - sim_res).abs() /
sim_res.abs()).reshape(-1).sort()[-20:])
def test_quantized_mobilnet():
x = nd.uniform(shape=(1, 3, 224, 224))
from models.quantized_mobilenet import mobilenet1_0 as my_mobilenet
my_net = my_mobilenet(pretrained=True)
my_res = my_net(x)
from gluoncv.model_zoo import mobilenet1_0 as ref_mobilenet
ref_net = ref_mobilenet(pretrained=True)
ref_res = ref_net(x)
sim_net = ref_mobilenet(pretrained=True)
convert_model(sim_net, exclude=[sim_net.features[0]])
qparams_init(sim_net)
sim_res = sim_net(x)
# print(">> Difference between my_quantized_mobilenet and simulated_mobilenet -- ")
# print(">> ", ((my_res - sim_res).abs() / sim_res.abs()).reshape(-1).sort()[-100:])
return (my_res.argsort(is_ascend=False)[0, :20],
ref_res.argsort(is_ascend=False)[0, :20],
sim_res.argsort(is_ascend=False)[0, :20])
def _sample_bernoulli(probability):
return nd.greater(probability, nd.uniform(shape=probability.shape))
def test_im2col_2D():
x = nd.uniform(shape=(2, 3, 5, 5))
print("Origin:")
print(x)
print("im2col_2D:")
print(_im2col_2D(nd, x, (3, 3), (1, 1), (1, 1)))
def train(epochs, ctx):
if isinstance(ctx, mx.Context):
ctx = [ctx]
if config.train_cfg.param_init:
init_func = getattr(mx.init, config.train_cfg.init)
net.initialize(init_func(), ctx=ctx, force_reinit=True)
else:
net.load_parameters(config.train_cfg.param_file, ctx=ctx)
summary(net, stat_name, nd.uniform(
shape=(1, 3, imgsize, imgsize), ctx=ctx[0]))
# net = nn.HybridBlock()
net.hybridize()
root = config.dir_cfg.dataset
train_data = gluon.data.DataLoader(
gluon.data.vision.CIFAR10(
root=root, train=True).transform_first(transform_train),
batch_size=batch_size, shuffle=True, last_batch='discard', num_workers=num_workers)
val_data = gluon.data.DataLoader(
gluon.data.vision.CIFAR10(
root=root, train=False).transform_first(transform_test),
batch_size=batch_size, shuffle=False, num_workers=num_workers)
trainer_arg = {'learning_rate': config.lr_cfg.lr,
'wd': config.lr_cfg.wd, 'lr_scheduler': lr_sch}
extra_arg = eval(config.lr_cfg.extra_arg)
trainer_arg.update(extra_arg)
trainer = gluon.Trainer(net.collect_params(), optimizer, trainer_arg)
if config.train_cfg.amp:
amp.init_trainer(trainer)
metric = mx.metric.Accuracy()
train_metric = mx.metric.RMSE()
loss_fn = gluon.loss.SoftmaxCrossEntropyLoss(
sparse_label=False if config.data_cfg.mixup else True)
train_history = TrainingHistory(['training-error', 'validation-error'])
# acc_history = TrainingHistory(['training-acc', 'validation-acc'])
loss_history = TrainingHistory(['training-loss', 'validation-loss'])
iteration = 0
best_val_score = 0
# print('start training')
sig_state.emit(1)
sig_pgbar.emit(0)
# signal.emit('Training')
for epoch in range(epochs):
tic = time.time()
train_metric.reset()
metric.reset()
train_loss = 0
num_batch = len(train_data)
alpha = 1
for i, batch in enumerate(train_data):
if epoch == 0 and iteration == 1 and config.save_cfg.profiler:
profiler.set_state('run')
is_profiler_run = True
if epoch == 0 and iteration == 1 and config.save_cfg.tensorboard:
sw.add_graph(net)
lam = np.random.beta(alpha, alpha)
if epoch >= epochs - 20 or not config.data_cfg.mixup:
lam = 1
data_1 = gluon.utils.split_and_load(
batch[0], ctx_list=ctx, batch_axis=0)
label_1 = gluon.utils.split_and_load(
batch[1], ctx_list=ctx, batch_axis=0)
if not config.data_cfg.mixup:
data = data_1
label = label_1
else:
data = [lam*X + (1-lam)*X[::-1] for X in data_1]
label = []
for Y in label_1:
y1 = label_transform(Y, classes)
y2 = label_transform(Y[::-1], classes)
label.append(lam*y1 + (1-lam)*y2)
with ag.record():
output = [net(X) for X in data]
loss = [loss_fn(yhat, y) for yhat, y in zip(output, label)]
if config.train_cfg.amp:
with ag.record():
with amp.scale_loss(loss, trainer) as scaled_loss:
ag.backward(scaled_loss)
# scaled_loss.backward()
else:
for l in loss:
l.backward()
trainer.step(batch_size)
train_loss += sum([l.sum().asscalar() for l in loss])
output_softmax = [nd.SoftmaxActivation(out) for out in output]
train_metric.update(label, output_softmax)
metric.update(label_1, output_softmax)
name, acc = train_metric.get()
if config.save_cfg.tensorboard:
sw.add_scalar(tag='lr', value=trainer.learning_rate,
global_step=iteration)
if epoch == 0 and iteration == 1 and config.save_cfg.profiler:
nd.waitall()
profiler.set_state('stop')
profiler.dump()
iteration += 1
sig_pgbar.emit(iteration)
if check_flag()[0]:
sig_state.emit(2)
while(check_flag()[0] or check_flag()[1]):
if check_flag()[1]:
print('stop')
return
else:
time.sleep(5)
print('pausing')
epoch_time = time.time() - tic
train_loss /= batch_size * num_batch
name, acc = train_metric.get()
_, train_acc = metric.get()
name, val_acc, _ = test(ctx, val_data)
# if config.data_cfg.mixup:
# train_history.update([acc, 1-val_acc])
# plt.cla()
# train_history.plot(save_path='%s/%s_history.png' %
# (plot_name, model_name))
# else:
train_history.update([1-train_acc, 1-val_acc])
plt.cla()
train_history.plot(save_path='%s/%s_history.png' %
(plot_name, model_name))
if val_acc > best_val_score:
best_val_score = val_acc
net.save_parameters('%s/%.4f-cifar-%s-%d-best.params' %
(save_dir, best_val_score, model_name, epoch))
current_lr = trainer.learning_rate
name, val_acc, val_loss = test(ctx, val_data)
logging.info('[Epoch %d] loss=%f train_acc=%f train_RMSE=%f\n val_acc=%f val_loss=%f lr=%f time: %f' %
(epoch, train_loss, train_acc, acc, val_acc, val_loss, current_lr, epoch_time))
loss_history.update([train_loss, val_loss])
plt.cla()
loss_history.plot(save_path='%s/%s_loss.png' %
(plot_name, model_name), y_lim=(0, 2), legend_loc='best')
if config.save_cfg.tensorboard:
sw._add_scalars(tag='Acc',
scalar_dict={'train_acc': train_acc, 'test_acc': val_acc}, global_step=epoch)
sw._add_scalars(tag='Loss',
scalar_dict={'train_loss': train_loss, 'test_loss': val_loss}, global_step=epoch)
sig_table.emit([epoch, train_loss, train_acc,
val_loss, val_acc, current_lr, epoch_time])
csv_writer.writerow([epoch, train_loss, train_acc,
val_loss, val_acc, current_lr, epoch_time])
csv_file.flush()
if save_period and save_dir and (epoch + 1) % save_period == 0:
net.save_parameters('%s/cifar10-%s-%d.params' %
(save_dir, model_name, epoch))
if save_period and save_dir:
net.save_parameters('%s/cifar10-%s-%d.params' %
(save_dir, model_name, epochs-1))
def forward(self, inputs, states_forward=None, states_backward=None):
inputs_reversed = nd.reverse(inputs, axis=2)
if not (states_forward and states_backward):
states_forward, states_backward = self.begin_state(batch_size=inputs.shape[1])
outputs_forward = []
outputs_backward = []
for i in range(self.num_layers):
if i == 0:
output, states_forward[i] = getattr(self, 'forward_lstm_{}'.format(i))(inputs, states_forward[i])
outputs_forward.append(output)
output, states_backward[i] = getattr(self, 'backward_lstm_{}'.format(i))(inputs_reversed, states_backward[i])
outputs_backward.append(output)
else:
output, states_forward[i] = getattr(self, 'forward_lstm_{}'.format(i))(outputs_forward[i-1], states_forward[i])
outputs_forward.append(output)
output, states_backward[i] = getattr(self, 'backward_lstm_{}'.format(i))(outputs_backward[i-1], states_backward[i])
outputs_backward.append(output)
return outputs_forward, states_forward, outputs_backward, states_backward
lstm = ElmoLSTMReverse('lstm', 3, 400, 1150, 0.4, 0.5, True)
lstm.collect_params().initialize()
states_forward, states_backward = lstm.begin_state(batch_size=80)
outputs_forward, states_forward, outputs_backward, states_backward = lstm(nd.uniform(-1, 1, (35,80,400)), states_forward, states_backward)
# 有3层用了最大池化,参数完全一样(pool_size=3, strides=2),目的使得卷积出 的尺寸和接下来的池化尺寸一样!!!
# NiN去除了容易造成过拟合的全连接输出层,替换成输出通道数等于标签类别数的NiN块和全局平均池化层,这种方式泛化能力相对低些,但是需要的显存大大降低
# 输出尺寸是输入的一半-(p,s)=(3,2),(pool_size,strides)
net = nn.Sequential()
net.add(NiN_block(channels=96, kernel_size=11, strides=4), nn.MaxPool2D(pool_size=3, strides=2),
NiN_block(channels=256, kernel_size=5, padding=2), nn.MaxPool2D(pool_size=3, strides=2),
NiN_block(channels=384, kernel_size=3, padding=1), nn.MaxPool2D(pool_size=3, strides=2), nn.Dropout(0.4),
# 类别数10,下面的全局平均池化自动接任10
NiN_block(channels=10, kernel_size=3, padding=1, strides=1),
# 全局平均池化-poolsize就是输入大小,即输出(_*_*1*1)
nn.GlobalAvgPool2D(),
# 将输出4D->2D(N,10)
nn.Flatten())
# Test
X = nd.uniform(shape=(1, 1, 224, 224))
net.initialize() # 默认初始化init=initializer.Uniform()
for layer in net:
X = layer(X)
print(layer.name, 'output shape:\t', X.shape)
# 读取数据
batch_size = 32
train_iter, test_iter = d2l.load_data_fashion_mnist(batch_size, resize=224)
# 重新初始化模型
ctx = d2l.try_gpu()
net.initialize(force_reinit=True, init=init.Xavier(), ctx=ctx) # 模型重新初始化
assert (len(value.shape) == 3)
d_model = query.shape[-1]
scores = nd.batch_dot(query, key, transpose_b=True) / math.sqrt(d_model)
if mask is not None:
val = nd.ones(scores.shape, ctx=cfg.ctx) * (-1e9)
scores = nd.where(mask == 1, scores, val)
p_attn = nd.softmax(scores, axis=-1)
if dropout is not None:
p_attn = dropout(p_attn)
return nd.batch_dot(p_attn, value), p_attn
if cfg.DEBUG_ON:
x = nd.uniform(0, 1, shape=(1, 9, 512))
out, attn = attention(x, x, x)
print('attention debug:')
print("\t input, out, attn: ", x.shape, out.shape, attn.shape)
# print(attn[0,:,:])
class MultiHeadedAttention(nn.Block):
def __init__(self, h, d_model, dropout=0.1):
super(MultiHeadedAttention, self).__init__()
assert (d_model % h == 0)
with self.name_scope():
self.d_k = d_model // h
self.h = h
self.linears_0 = nn.Dense(in_units=d_model,
def fuzzy_one_hot(arr, size):
x = arr.reshape((-1, ))
return nd.where(nd.one_hot(x, size),
nd.uniform(low=0.7, high=1.2, shape=(x.shape[0], size), ctx=x.context),
nd.uniform(low=0.0, high=0.3, shape=(x.shape[0], size), ctx=x.context))
def forward(self, x, y, sample_prob=None):
if sample_prob is not None:
self.sample_prob = sample_prob
batch_size = x.shape[0]
state = self.init_hidden(batch_size, self.ctx)
outputs_pgm = []
outputs_param = []
seq = y
for i in range(seq.shape[1]):
if i == 0:
xt = x
else:
if i >= 1 and self.sample_prob > 0:
#print("x.shape:",x.shape)
sample_prob = nd.uniform(
0, 1, shape=(batch_size),
ctx=self.ctx) #sample_prob.shape (10,)
sample_mask = sample_prob < self.sample_prob
#print("sample_mask:",sample_mask)
#print("sample_mask.sum:",sample_mask.sum().asscalar())
if sample_mask.sum() == 0:
it1 = seq[:, i - 1]
else:
sample_ind = sample_mask != 0
#print("sample_ind:",sample_ind)
it1 = seq[:, i - 1] #it1.shape : (10,)
#print("it1:",it1.shape)
#print("output_prog:",outputs_pgm[-1])
prob_prev = nd.exp(outputs_pgm[-1])
#print("prob_pre:",prob_prev)
temp = nd.random.multinomial(
prob_prev, 1).reshape(-1).astype('int64')
#print("prob_prev:",nd.argmax(prob_prev,axis=1).astype('int64')==temp)
#print("temp",temp,"\n it1:",it1)
it1 = nd.where(sample_ind, temp, it1).astype('float32')
else:
#print("obtain last ground truth")
it1 = seq[:, i - 1].copy()
xt = self.pgm_embed(it1)
#print("xt after embed:",xt)
#print("xt :",xt)
output, state = self.core(xt.expand_dims(axis=0), state)
pgm_feat1 = nd.relu(self.logit1(output.squeeze(0)))
pgm_feat2 = self.logit2(pgm_feat1)
pgm_score = nd.log_softmax(pgm_feat2, axis=1)
trans_prob = nd.softmax(pgm_feat2, axis=1).detach()
param_feat1 = nd.relu(self.regress1(output.squeeze(0)))
param_feat2 = nd.concat(trans_prob, param_feat1, dim=1)
param_score = self.regress2(param_feat2)
param_score = param_score.reshape(batch_size, self.vocab_size + 1,
self.max_param)
#index = nd.argmax(trans_prob, axis = 1)
index = seq[:, i]
index = index.expand_dims(axis=1).expand_dims(axis=2).broadcast_to(
shape=(batch_size, 1, self.max_param)).detach()
param_score = nd.pick(param_score, index, 1)
outputs_pgm.append(pgm_score)
outputs_param.append(param_score)
outputs_pgm = [_.expand_dims(axis=1) for _ in outputs_pgm]
outputs_param = [_.expand_dims(axis=1) for _ in outputs_param]
pgms = outputs_pgm[0]
params = outputs_param[0]
for i in range(1, len(outputs_pgm)):
pgms = nd.concat(pgms, outputs_pgm[i], dim=1)
params = nd.concat(params, outputs_param[i], dim=1)
#print("params", params.shape)
#rint("pgm", pgms.shape)
return [pgms, params]
def randint(low=0., high=1., shape=(1, ), ctx=None, dtype='int32'):
return nd.uniform(low=low, high=high, shape=shape, ctx=ctx).astype(dtype)
def fuzzy_one_hot(arr, size):
x = arr.reshape((-1, ))
return nd.where(
nd.one_hot(x, size),
nd.uniform(low=0.7, high=1.2, shape=(x.shape[0], size), ctx=x.context),
nd.uniform(low=0.0, high=0.3, shape=(x.shape[0], size), ctx=x.context))
def randint(low=0., high=1., shape=(1, ), ctx=None, dtype='int32'):
return nd.uniform(low=low, high=high, shape=shape, ctx=ctx).astype(dtype)
x = self.conv2(x_)
x = x_ + x
x_ = self.conv3(x)
x = F.concat(x, x_, dim=1)
x = self.pool(x)
x = self.fc1(x)
x = self.fc2(x)
return x
if __name__ == "__main__":
root = "../models/my_testmodel/"
# Generate model and initialize
model = TestModel()
model.initialize()
input_ = nd.uniform(shape=(1, 3, 224, 224))
_ = model(input_)
# Save parameters
model.save_parameters(root + "my_testmodel.param")
# Save input
nd.save(root + "input.nd", input_)
save(root + "input.dat", input_.asnumpy())
# Save conv1
conv = model.conv1
save(root + "conv1.weight.dat", conv.weight.data().asnumpy())
# Save conv2
conv = model.conv2
save(root + "conv2.weight.dat", conv.weight.data().asnumpy())
save(root + "conv2.bias.dat", conv.bias.data().asnumpy())
# Save conv3
conv = model.conv3
