def __init__(self, name):
with open("{}".format(name), "rb") as f:
self.net = load_network(f)
self.test_func = Function().compile([self.net.outputs[0], self.net.loss_var])
with open("/unsullied/sharefs/liuyanyi02/lyy/CIFAR/meanstd.data", "rb") as f:
self.mean, self.std = pickle.load(f)
self.mean = np.array(self.mean).reshape(3, 32, 32)
self.std = np.array(self.std).reshape(3, 32, 32)
def bn_post_process(model_file: str, save_model_file: str, data):
with TrainingEnv(name=model_file + "bn_post_proc", part_count=2) as env:
net = load_network(open(model_file, "rb"))
#loss_func = env.make_func_from_loss_var(net.loss_var, "val", train_state = False)
bn_oprs = [
opr for opr in net.loss_visitor.all_oprs
if isinstance(opr, BatchNormalization)
]
bn_inputs = [opr.inputs[0] for opr in bn_oprs]
mean_Esqr_nodes = []
for i in bn_inputs:
if i.partial_shape.ndim == 2:
mean = i.mean(axis=0).reshape((1, -1))
mean.vflags.data_parallel_reduce_method = 'sum'
Esqr = (i**2).mean(axis=0).reshape((1, -1))
Esqr.vflags.data_parallel_reduce_method = 'sum'
if i.partial_shape.ndim == 4:
mean = i.mean(axis=3).mean(axis=2).mean(axis=0).reshape(
(1, -1))
mean.vflags.data_parallel_reduce_method = 'sum'
Esqr = (i**2).mean(axis=3).mean(axis=2).mean(axis=0).reshape(
(1, -1))
Esqr.vflags.data_parallel_reduce_method = 'sum'
mean_Esqr_nodes.append(mean)
mean_Esqr_nodes.append(Esqr)
func = Function().compile(mean_Esqr_nodes)
for i in range(len(bn_oprs)):
opr = bn_oprs[i]
layer_mean, layer_var = _get_dataset_mean_var(data, func, i)
if layer_mean.ndim == 0:
layer_mean = layer_mean.reshape((1, ))
if layer_var.ndim == 0:
layer_var = layer_var.reshape((1, ))
state = opr.State(channels=layer_mean.shape[0],
val=[layer_mean, layer_var, 1])
state.owner_opr_type = type(opr)
opr.set_opr_state(state)
opr.freezed = True
env.register_checkpoint_component("network", net)
env.save_checkpoint(save_model_file)
def test(net=None):
if net == None:
net = load_network(open("./data/resnet110.data", "rb"))
test_func = Function().compile(net.outputs[0])
"""
from megskull.network import NetworkVisitor
visitor = NetworkVisitor(net.loss_var)
for i in visitor.all_oprs:
print(i)
print("input = ", i.inputs)
print("output = ", i.outputs)
a = np.array(i.outputs[0].get_value())
print(a)
input()
a = np.array(visitor.all_oprs_dict["conv1:W"].get_value())
print(a)
print("mean = ", np.mean(a, axis = 0))
print("std = ", np.std(a, axis = 0))
exit()
"""
dic = load_data("/home/liuyanyi02/CIFAR/cifar-10-batches-py/test_batch")
data = dic[b'data']
label = dic[b'labels']
data = data.astype(np.float32)
import pickle
with open("meanstd.data", "rb") as f:
mean, std = pickle.load(f)
data = (data - mean) / std
data = np.resize(data, (10000, 3, 32, 32))
data = data.astype(np.float32)
"""
import cv2
for i in range(10):
img = data[i].transpose(1, 2, 0)
img = img[:,::-1,:]
cv2.imshow('x', img)
cv2.waitKey(0)
"""
#data = data.astype(np.float32)
#data = (data - 128) / 256
pred = test_func(data=data)
print(np.array(pred).shape)
pred = np.argmax(np.array(pred), axis=1)
acc = (np.array(pred) == np.array(label)).mean()
print(acc)
def myw_test():
d40_MY = NN("./data/r20_MY.data")
net = d40_MY.net
outputs = []
visitor = NetworkVisitor(net.loss_var)
for i in visitor.all_oprs:
if "fc1" in i.name and ":W" not in i.name and ":b" not in i.name:
outputs.append(i)
func = Function().compile(outputs)
data, labels = load_CIFAR_data()
batch = data[:128]
batch = batch.reshape(128, 3, 32, 32)
mean, std = d40_MY.mean, d40_MY.std
batch = (batch - mean) / std
outputs_weights = func(data=batch)
for i in outputs_weights:
print(i.shape)
w = i[0]
w = w.reshape(-1, 4, 4)
print(w)
input()
def test():
data, labels = load_CIFAR_data()
p120 = NN("data/p120.data")
net = p120.net
loss = net.loss_var
visitor = NetworkVisitor(loss)
inp = []
for i in visitor.all_oprs:
if "data" in i.name:
inp.append(i)
if "conv" in i.name and ":" not in i.name:
inp.append(i)
print(inp)
grad = []
out = []
for i in inp:
grad.append(O.Grad(loss, i))
out.append(i)
F = Function()
F._env.flags.train_batch_normalization = True
func = F.compile(grad)
F = Function()
F._env.flags.train_batch_normalization = True
func1 = F.compile(out)
batch = data[:128]
batch = batch.reshape(128, 3, 32, 32)
mean, std = p120.mean, p120.std
batch = (batch - mean) / std
label = labels[:128]
grad_out = func(data=batch, label=label)
lay_out = func1(data=batch, label=label)
idx = 0
grad_list = []
for i, j in zip(grad_out, lay_out):
print(i.shape, idx)
idx += 1
f = i.flatten()
print("grad")
print(f)
print(np.mean(f), np.std(f))
grad_list.append(np.std(f))
print("val")
h = j.flatten()
print(h)
print(np.mean(h), np.std(j))
pickle.dump(grad_list, open("p120_norelu_grad.data", "wb"))
"""
def init(net, batch):
visitor = NetworkVisitor(net.loss_var)
lisk = []
lisb = []
for i in visitor.all_oprs:
if ":k" in i.name and "bnaff" in i.name:
lisk.append(i)
if ":b" in i.name and "bnaff" in i.name:
lisb.append(i)
for i, k, b in zip(range(len(lisk)), lisk, lisb):
func = Function().compile(net.outputs)
outputs = func(data=batch['data'])
t = outputs[1 + i]
mean = t.mean(axis=3).mean(axis=2).mean(axis=0)
std = ((t - mean[np.newaxis, :, np.newaxis, np.newaxis])**2).mean(
axis=3).mean(axis=2).mean(axis=0)**0.5
nk = O.ParamProvider("new" + k.name, 1.0 / std)
nb = O.ParamProvider("new" + b.name, -mean / std)
visitor.replace_vars([(k, nk), (b, nb)], copy=False)
visitor = NetworkVisitor(net.loss_var)
for i in visitor.all_oprs:
print(i)
return net
train_func = env.make_func_from_loss_var(net.loss_var, "train", train_state = True)
lr = 0.1 * num_GPU
optimizer = Momentum(lr, 0.9)
optimizer(train_func)
#train_func.comp_graph.share_device_memory_with(valid_func.comp_graph)
dic = {
"loss": net.loss_var,
"pre_loss": preloss,
"outputs": net.outputs[0]
}
train_func.compile(dic)
valid_func = Function().compile(net.outputs[0])
env.register_checkpoint_component("network", net)
env.register_checkpoint_component("opt_state", train_func.optimizer_state)
tr_p = InputPipe("lyy.CIFAR10.train", buffer_size = 1000)
va_p = InputPipe("lyy.CIFAR10.valid", buffer_size = 1000)
epoch = 0
EPOCH_NUM = 50000 // minibatch_size
i = 0
max_acc = 0
ORI_IT = 64000
BN_IT = 10000
TOT_IT = ORI_IT + BN_IT
C = CIFAR_test()
from megskull.network import NetworkVisitor
visitor = NetworkVisitor(net.loss_var)
offsets = []
locs = []
for i in visitor.all_oprs:
print(i, i.name)
if "Astype" in i.name:
locs.append(i)
if i.name == "conv2offsetx":
offsets.append(i)
print("A")
print(len(locs))
locs = locs[::4]
outs = [net.outputs[0]] + locs
test_func = Function().compile(outs)
outs1 = offsets
offs_func = Function().compile(outs1)
def load_data(name):
import pickle
with open(name, "rb") as fo:
dic = pickle.load(fo, encoding="bytes")
return dic
dic = load_data("/home/liuyanyi02/CIFAR/cifar-10-batches-py/test_batch")
data = dic[b'data']
label = dic[b'labels']
from meghair.utils.io import load_network
from megskull.graph import Function
import numpy as np
from numpy.linalg import svd
net = load_network(open("./data/plain30_orth.data", "rb"))
func = Function().compile(net.outputs)
def load_data(name):
import pickle
with open(name, "rb") as fo:
dic = pickle.load(fo, encoding="bytes")
return dic
dic = load_data("/home/liuyanyi02/CIFAR/cifar-10-batches-py/test_batch")
data = dic[b'data']
label = dic[b'labels']
data = data.astype(np.float32)
import pickle
with open("meanstd.data", "rb") as f:
mean, std = pickle.load(f)
data = (data - mean) / std
data = np.resize(data, (10000, 3, 32, 32))
data = data.astype(np.float32)
idx = np.random.randint(data.shape[0], size=3000)
data = [data[i] for i in idx]
#!/usr/bin/env mdl
from megbrain.config import set_default_device
from megskull.graph import Function
from neupeak.utils.cli import load_network
import dataset
import cv2
import numpy as np
set_default_device('cpu0')
net = load_network(
'/home/zhaojing/vehicle_pose/config/xception145/train_log/models/latest')
classify = Function().compile(net.outputs[0])
test_dataset = dataset.get('test')
x = test_dataset.get_epoch_minibatch_iter()
correct = [0, 0]
total_label = [0, 0]
total_pred = [0, 0]
for data in x:
out = classify(data.data)
#total += data.label.size
for i in range(0, data.label.size):
total_pred[out[i].argmax()] += 1
total_label[data.label[i]] += 1
if out[i].argmax() == data.label[i]:
correct[data.label[i]] += 1
accuracy = [0, 0]
from megskull.graph import Function
from megskull.optimizer.momentum import Momentum
import network
from megskull.graph import FpropEnv
from meghair.utils.io import dump
func = OptimizableFunc.make_from_loss_var(network.loss)
Momentum(learning_rate=1e-2, momentum=0.9)(func)
func.compile(network.loss)
#env = FpropEnv()
#pred_mgb = env.get_mgbvar(network.pred)
#func_test = env.comp_graph.compile_outonly(pred_mgb)
func_test = Function().compile(network.pred)
import pickle
import gzip
import numpy as np
train_set, valid_set, test_set = pickle.load(gzip.open("mnist.pkl.gz", "rb"),
encoding="latin1")
minibatch_size = network.minibatch_size
l = len(train_set[0])
epoch = 0
for i in range(100000):
j = i % (l // minibatch_size)
minibatch = train_set[0][j * minibatch_size:(j + 1) * minibatch_size]