def convBN(inmaps,
outmaps,
size,
stride,
pad,
name,
actInplace,
bnInplace,
scheme,
typ="bn"):
block = Sequential()
if typ == "bn":
names = ["conv_%s" % name, "bn_%s" % name, "relu_%s" % name]
elif typ == "v3":
names = ["%s_conv2d" % name, "%s_batchnorm" % name, "%s_relu" % name]
else:
raise ValueError("Unrecognized convBN type")
block.append(
Conv2D(inmaps,
outmaps,
size,
stride,
pad,
useBias=False,
initscheme=scheme,
name=names[0]))
block.append(BatchNorm2D(outmaps, inplace=bnInplace, name=names[1]))
block.append(Activation(relu, inplace=actInplace, name=names[2]))
return block
def block(idx,
inmaps,
outmaps,
sizeconv,
strideconv,
initscheme,
actInPlace,
sizepool=2,
stridepool=2,
addMaxpool=True):
assert len(inmaps) == len(outmaps) == len(sizeconv) == len(
strideconv) == len(idx), "lengths must be the same size"
seq = Sequential()
for i in range(len(inmaps)):
seq.append(
Conv2D(inmaps=inmaps[i],
outmaps=outmaps[i],
size=sizeconv[i],
pad=sizeconv[i] // 2,
stride=strideconv[i],
initscheme=initscheme,
dilation=1,
useBias=True,
name="conv%s" % idx[i]))
seq.append(Activation(leakyRelu, inplace=actInPlace, args=(0.01, )))
if addMaxpool:
seq.append(
MaxPool2D(size=sizepool,
stride=stridepool,
name="conv%s_pool" % idx[-1]))
return seq
def residMiniBlock(inmaps, outmaps, size, stride, pad, blockname, mininame,
addAct, actInplace, bnInplace, initscheme):
block = Sequential()
block.append(
Conv2D(inmaps,
outmaps,
size,
stride=stride,
pad=pad,
useBias=False,
initscheme=initscheme,
name="res%s_branch%s" % (blockname, mininame)))
block.append(
BatchNorm2D(outmaps,
name="bn%s_branch%s" % (blockname, mininame),
inplace=bnInplace))
if addAct:
block.append(
Activation(relu,
inplace=actInplace,
name="res%s_branch%s_relu" % (blockname, mininame)))
return block
def buildBranch(fHeight, sentlength, branchMaps, embsize):
seq = Sequential()
seq.append(Conv2D(1, outmaps=branchMaps, size=(fHeight, embsize)))
seq.append(MaxPool2D(size=(sentlength - fHeight + 1, 1)))
seq.append(Reshape((-1, branchMaps)))
return seq
def buildSmallBlock(inplace=True):
block = Sequential()
block.append(Replicate(3))
left = buildSmallBranch(inplace=inplace, num=1)
right = buildSmallBranch(inplace=inplace, num=2)
shortcut = Sequential().append(Identity())
block.append(Parallel().append(left).append(right).append(shortcut))
block.append(Concat(axis=1, name="concat_stage2"))
return block
def buildBall(stage=2, inplace=True):
ball = Sequential()
ball.append(Replicate(2))
left = buildBranch(stage=stage, num=1, inplace=inplace)
right = buildBranch(stage=stage, num=2, inplace=inplace)
ball.append(Parallel().append(left).append(right))
ball.append(Concat(axis=1))
return ball
def buildBigBlock(stage=2, prenet=None, inplace=True):
block = Sequential()
block.append(Replicate(2))
shortcut = Sequential().append(Identity())
if prenet is None:
ball = buildBall(stage=stage, inplace=inplace)
else:
ball = prenet
ball.extend(buildBall(stage=stage, inplace=inplace))
block.append(Parallel().append(ball).append(shortcut))
block.append(Concat(axis=1, name="concat_stage%d" % (stage + 1)))
return block
def unittest():
from PuzzleLib.Containers.Sequential import Sequential
from PuzzleLib.Modules import Linear, Activation, sigmoid, Identity, Concat
data1 = gpuarray.to_gpu(np.random.randn(128, 128).astype(np.float32))
data2 = gpuarray.to_gpu(np.random.randn(128, 16).astype(np.float32))
data3 = gpuarray.to_gpu(np.random.randn(128, 32).astype(np.float32))
seq = Sequential()
seq.append(Linear(128, 64))
seq.append(Activation(sigmoid))
parallel = Parallel()
parallel.append(seq)
parallel.append(Identity())
parallel.append(Identity())
concat = Concat(axis=1)
parallel([data1, data2, data3])
concat(parallel.data)
assert np.allclose(data2.get(), concat.data.get()[:, 64:64 + 16])
grad = gpuarray.to_gpu(np.random.randn(128, 112).astype(np.float32))
concat.backward(grad)
parallel.backward(concat.grad)
assert np.allclose(grad.get()[:, 64:64 + 16], parallel.grad[1].get())
parallel = parallel[::2]
parallel([data1, data3])
def tower(towername,
names,
maps,
sizes,
strides,
pads,
act,
bn,
scheme,
typ="bn"):
block = Sequential()
lvlnames = ["%s_%s" % (towername, name) for name in names]
for i, name in enumerate(lvlnames):
if "pool" in name:
block.append(pool2D(sizes[i], strides[i], pads[i], name=names[i]))
else:
act = False if i == len(names) - 1 else act
block.extend(
convBN(maps[i], maps[i + 1], sizes[i], strides[i], pads[i],
lvlnames[i], act, bn, scheme, typ))
return block
def factorBlock(inmaps, b1m, b2m, b3m, b4m, name, act, bn, scheme):
block = Sequential()
b1towername, b1names = name, ["conv"]
b2towername, b2names = "%s_tower" % name, ["conv", "conv_1", "conv_2"]
b3towername, b3names = "%s_tower_1" % name, [
"conv", "conv_1", "conv_2", "conv_3", "conv_4"
]
b4towername, b4names = "%s_tower_2" % name, ["avg_pool", "conv"]
branch1 = tower(b1towername,
b1names, [inmaps] + b1m, [1], [1], [0],
act=act,
bn=bn,
scheme=scheme,
typ="v3")
branch2 = tower(b2towername,
b2names, [inmaps] + b2m, [1, (1, 7), (7, 1)], [1, 1, 1],
[0, (0, 3), (3, 0)],
act=act,
bn=bn,
scheme=scheme,
typ="v3")
branch3 = tower(b3towername,
b3names, [inmaps] + b3m,
[1, (7, 1), (1, 7), (7, 1), (1, 7)], [1, 1, 1, 1, 1],
[0, (3, 0), (0, 3), (3, 0), (0, 3)],
act=act,
bn=bn,
scheme=scheme,
typ="v3")
branch4 = tower(b4towername,
b4names, [inmaps, inmaps] + b4m, [3, 1], [1, 1], [1, 0],
act=act,
bn=bn,
scheme=scheme,
typ="v3")
block.append(Replicate(times=4))
block.append(Parallel().append(branch1).append(branch2).append(
branch3).append(branch4))
block.append(Concat(axis=1, name="ch_concat_%s_chconcat" % name))
return block
def gruTest(): batchsize, seqlen, insize = 5, 6, 4 hsize = 3 seq = Sequential(name="gru") seq.append(SwapAxes(0, 1)) seq.append(RNN(insize, hsize, mode="gru", getSequences=True, hintBatchSize=batchsize)) seq.append(SwapAxes(0, 1)) data = gpuarray.to_gpu(np.random.randn(batchsize, seqlen, insize).astype(np.float32)) engine = buildRTEngine(seq, data.shape, savepath="../TestData", dtype=DataType.float32) outdata = seq(data) enginedata = engine(data) assert np.allclose(outdata.get(), enginedata.get())
def v3ShrinkBlock(inmaps, b1m, b2m, name, act, bn, scheme):
block = Sequential()
b1towername, b1names = "%s_tower" % name, ["conv", "conv_1"]
b2towername, b2names = "%s_tower_1" % name, [
"conv", "conv_1", "conv_2", "conv_3"
]
b3towername, b3names = name, ["max_pool"]
branch1 = tower(b1towername,
b1names, [inmaps] + b1m, [1, 3], [1, 2], [0, 0],
act=act,
bn=bn,
scheme=scheme,
typ="v3")
branch2 = tower(b2towername,
b2names, [inmaps] + b2m, [1, (1, 7), (7, 1), 3],
[1, 1, 1, 2], [0, (0, 3), (3, 0), 0],
act=act,
bn=bn,
scheme=scheme,
typ="v3")
branch3 = tower(b3towername,
b3names, [inmaps, inmaps], [3], [2], [0],
act=act,
bn=bn,
scheme=scheme,
typ="v3")
block.append(Replicate(times=3))
block.append(Parallel().append(branch1).append(branch2).append(branch3))
block.append(Concat(axis=1, name="ch_concat_%s_chconcat" % name))
return block
def bnShrinkBlock(inmaps,
b1m,
b2m,
name,
act,
bn,
scheme,
b1deep=True,
pad=1,
typ="bn"):
block = Sequential()
if typ == "bn":
if b1deep:
b1towername, b1names = name, ["3x3_reduce", "3x3"]
else:
b1towername, b1names = name, ["3x3"]
b2towername, b2names = name, [
"double_3x3_reduce", "double_3x3_0", "double_3x3_1"
]
b3towername, b3names = name, ["max_pool"]
elif typ == "v3":
if b1deep:
b1towername, b1names = name, ["conv"]
else:
b1towername, b1names = name, ["conv"]
b2towername, b2names = "%s_tower" % name, ["conv", "conv_1", "conv_2"]
b3towername, b3names = name, ["max_pool"]
else:
raise ValueError("Unrecognized block type")
if b1deep:
branch1 = tower(b1towername,
b1names, [inmaps] + b1m, [1, 3], [1, 2], [0, pad],
act=act,
bn=bn,
scheme=scheme,
typ=typ)
else:
branch1 = tower(b1towername,
b1names, [inmaps] + b1m, [3], [2], [pad],
act=act,
bn=bn,
scheme=scheme,
typ=typ)
branch2 = tower(b2towername,
b2names, [inmaps] + b2m, [1, 3, 3], [1, 1, 2], [0, 1, pad],
act=act,
bn=bn,
scheme=scheme,
typ=typ)
branch3 = tower(b3towername,
b3names, [inmaps, inmaps], [3], [2], [pad],
act=act,
bn=bn,
scheme=scheme,
typ=typ)
block.append(Replicate(times=3))
block.append(Parallel().append(branch1).append(branch2).append(branch3))
block.append(Concat(axis=1, name="ch_concat_%s_chconcat" % name))
return block
def bnBlock(inmaps,
b1m,
b2m,
b3m,
b4m,
name,
act,
bn,
scheme,
b2size=3,
b2pad=1,
typ="bn"):
block = Sequential()
if typ == "bn":
b1towername, b1names = name, ["1x1"]
b2towername, b2names = name, ["3x3_reduce", "3x3"]
b3towername, b3names = name, [
"double_3x3_reduce", "double_3x3_0", "double_3x3_1"
]
b4towername, b4names = name, ["avg_pool", "proj"]
elif typ == "v3":
b1towername, b1names = name, ["conv"]
b2towername, b2names = "%s_tower" % name, ["conv", "conv_1"]
b3towername, b3names = "%s_tower_1" % name, [
"conv", "conv_1", "conv_2"
]
b4towername, b4names = "%s_tower_2" % name, ["avg_pool", "conv"]
else:
raise ValueError("Unrecognized block type")
branch1 = tower(b1towername,
b1names, [inmaps] + b1m, [1],
strides=[1],
pads=[0],
act=act,
bn=bn,
scheme=scheme,
typ=typ)
branch2 = tower(b2towername,
b2names, [inmaps] + b2m, [1, b2size],
strides=[1, 1],
pads=[0, b2pad],
act=act,
bn=bn,
scheme=scheme,
typ=typ)
branch3 = tower(b3towername,
b3names, [inmaps] + b3m, [1, 3, 3],
strides=[1, 1, 1],
pads=[0, 1, 1],
act=act,
bn=bn,
scheme=scheme,
typ=typ)
branch4 = tower(b4towername,
b4names, [inmaps, inmaps] + b4m, [3, 1],
strides=[1, 1],
pads=[1, 0],
act=act,
bn=bn,
scheme=scheme,
typ=typ)
block.append(Replicate(times=4))
block.append(Parallel().append(branch1).append(branch2).append(
branch3).append(branch4))
block.append(Concat(axis=1, name="ch_concat_%s_chconcat" % name))
return block
def loadNet(name="", inplace=True, modelpath=None):
net = Sequential(name)
net.append(Conv2D(3, 64, 3, pad=1, initscheme="none", name="conv1_1"))
net.append(Activation(relu, name="relu1_1", inplace=inplace))
net.append(Conv2D(64, 64, 3, pad=1, initscheme="none", name="conv1_2"))
net.append(Activation(relu, name="relu1_2", inplace=inplace))
net.append(MaxPool2D(name="pool1_stage1"))
net.append(Conv2D(64, 128, 3, pad=1, initscheme="none", name="conv2_1"))
net.append(Activation(relu, name="relu2_1", inplace=inplace))
net.append(Conv2D(128, 128, 3, pad=1, initscheme="none", name="conv2_2"))
net.append(Activation(relu, name="relu2_2", inplace=inplace))
net.append(MaxPool2D(name="pool2_stage1"))
net.append(Conv2D(128, 256, 3, pad=1, initscheme="none", name="conv3_1"))
net.append(Activation(relu, name="relu3_1", inplace=inplace))
net.append(Conv2D(256, 256, 3, pad=1, initscheme="none", name="conv3_2"))
net.append(Activation(relu, name="relu3_2", inplace=inplace))
net.append(Conv2D(256, 256, 3, pad=1, initscheme="none", name="conv3_3"))
net.append(Activation(relu, name="relu3_3", inplace=inplace))
net.append(Conv2D(256, 256, 3, pad=1, initscheme="none", name="conv3_4"))
net.append(Activation(relu, name="relu3_4", inplace=inplace))
net.append(MaxPool2D(name="pool3_stage1"))
net.append(Conv2D(256, 512, 3, pad=1, initscheme="none", name="conv4_1"))
net.append(Activation(relu, name="relu4_1", inplace=inplace))
net.append(Conv2D(512, 512, 3, pad=1, initscheme="none", name="conv4_2"))
net.append(Activation(relu, name="relu4_2", inplace=inplace))
net.append(
Conv2D(512, 256, 3, pad=1, initscheme="none", name="conv4_3_CPM"))
net.append(Activation(relu, name="relu4_3_CPM"))
net.append(
Conv2D(256, 128, 3, pad=1, initscheme="none", name="conv4_4_CPM"))
net.append(Activation(relu, name="relu4_4_CPM"))
block2 = buildSmallBlock(inplace=inplace)
block3 = buildBigBlock(stage=2, prenet=block2, inplace=inplace)
block4 = buildBigBlock(stage=3, prenet=block3, inplace=inplace)
block5 = buildBigBlock(stage=4, prenet=block4, inplace=inplace)
block6 = buildBigBlock(stage=5, prenet=block5, inplace=inplace)
net.extend(block6)
net.append(Replicate(2))
net.append(Parallel().append(buildBranch(stage=6, num=2,
inplace=inplace)).append(
buildBranch(stage=6,
num=1,
inplace=inplace)))
net.append(Concat(axis=1))
if modelpath is not None:
net.load(modelpath, assumeUniqueNames=True)
return net
def trainHardTest(optCls, dtype, *args, **kwargs):
from PuzzleLib.Containers.Sequential import Sequential
from PuzzleLib.Modules.Conv2D import Conv2D
from PuzzleLib.Modules.BatchNorm2D import BatchNorm2D
from PuzzleLib.Modules.Activation import Activation, relu
from PuzzleLib.Modules.Cast import Cast
from PuzzleLib.Cost.MSE import MSE
seq = Sequential()
seq.append(Conv2D(4, 8, 5, pad=1))
seq.append(BatchNorm2D(8))
seq.append(Activation(relu))
seq.append(Conv2D(8, 16, 5, pad=1))
seq.calcMode(dtype)
seq.append(Cast(intype=dtype, outtype=np.float32))
optimizer = optCls(*args, **kwargs)
optimizer.setupOn(seq, useGlobalState=True)
mse = MSE()
data = gpuarray.to_gpu(np.random.randn(4, 4, 5, 5).astype(dtype))
target = gpuarray.to_gpu(np.random.randn(4, 16, 1, 1).astype(np.float32))
for i in range(200):
error, grad = mse(seq(data), target)
optimizer.zeroGradParams()
seq.backward(grad)
optimizer.update()
if (i + 1) % 5 == 0:
print("Iteration #%d error: %s" % (i + 1, error))
def expandBlock(inmaps, b1m, b2m, b3m, b4m, name, act, bn, scheme, pool="avg"):
block = Sequential()
b1towername, b1names = name, ["conv"]
b2towername, b2names, b2sub1names, b2sub2names = "%s_tower" % name, [
"conv"
], ["mixed_conv"], ["mixed_conv_1"]
b3towername, b3names, b3sub1names, b3sub2names = "%s_tower_1" % name, [
"conv", "conv_1"
], ["mixed_conv"], ["mixed_conv_1"]
branch1 = tower(b1towername,
b1names, [inmaps] + b1m, [1], [1], [0],
act=act,
bn=bn,
scheme=scheme,
typ="v3")
branch2 = tower(b2towername,
b2names, [inmaps, b2m[0]], [1], [1], [0],
act=act,
bn=bn,
scheme=scheme,
typ="v3")
branch2sub1 = tower(b2towername,
b2sub1names, [b2m[0], b2m[1]], [(1, 3)], [1], [(0, 1)],
act=act,
bn=bn,
scheme=scheme,
typ="v3")
branch2sub2 = tower(b2towername,
b2sub2names, [b2m[0], b2m[2]], [(3, 1)], [1], [(1, 0)],
act=act,
bn=bn,
scheme=scheme,
typ="v3")
branch2.append(Replicate(times=2))
branch2.append(Parallel().append(branch2sub1).append(branch2sub2))
branch3 = tower(b3towername,
b3names, [inmaps, b3m[0], b3m[1]], [1, 3], [1, 1], [0, 1],
act=act,
bn=bn,
scheme=scheme,
typ="v3")
branch3sub1 = tower(b3towername,
b3sub1names, [b3m[1], b3m[2]], [(1, 3)], [1], [(0, 1)],
act=act,
bn=bn,
scheme=scheme,
typ="v3")
branch3sub2 = tower(b3towername,
b3sub2names, [b3m[1], b3m[3]], [(3, 1)], [1], [(1, 0)],
act=act,
bn=bn,
scheme=scheme,
typ="v3")
branch3.append(Replicate(times=2))
branch3.append(Parallel().append(branch3sub1).append(branch3sub2))
if pool == "avg":
branch4 = tower("%s_tower_2" % name, ["avg_pool", "conv"],
[inmaps, inmaps] + b4m, [3, 1], [1, 1], [1, 0],
act=act,
bn=bn,
scheme=scheme,
typ="v3")
elif pool == "max":
branch4 = tower("%s_tower_2" % name, ["max_pool", "conv"],
[inmaps, inmaps] + b4m, [3, 1], [1, 1], [1, 0],
act=act,
bn=bn,
scheme=scheme,
typ="v3")
else:
raise ValueError("Unrecognized block type")
block.append(Replicate(times=4))
block.append(Parallel().append(branch1).append(branch2).append(
branch3).append(branch4))
block.append(ToList())
block.append(Concat(axis=1, name="ch_concat_%s_chconcat" % name))
return block
def buildSmallBranch(inplace=True, num=1):
branch = Sequential()
branch.append(
Conv2D(128,
128,
3,
pad=1,
initscheme="none",
name="conv5_1_CPM_L%d" % num))
branch.append(
Activation(relu, inplace=inplace, name="relu5_1_CPM_L%d" % num))
branch.append(
Conv2D(128,
128,
3,
pad=1,
initscheme="none",
name="conv5_2_CPM_L%d" % num))
branch.append(
Activation(relu, inplace=inplace, name="relu5_2_CPM_L%d" % num))
branch.append(
Conv2D(128,
128,
3,
pad=1,
initscheme="none",
name="conv5_3_CPM_L%d" % num))
branch.append(
Activation(relu, inplace=inplace, name="relu5_3_CPM_L%d" % num))
branch.append(
Conv2D(128, 512, 1, initscheme="none", name="conv5_4_CPM_L%d" % num))
branch.append(
Activation(relu, inplace=inplace, name="relu5_4_CPM_L%d" % num))
branch.append(
Conv2D(512,
19 * (3 - num),
1,
initscheme="none",
name="conv5_5_CPM_L%d" % num))
return branch
def buildNet(vocabulary,
branches,
w2v,
sentlength,
embsize,
wscale,
dim=2,
branchMaps=100,
name="sentinet"):
def onVocabulary(W):
W[0] = np.zeros((1, embsize), dtype=np.float32)
arrayPOS = [
"", "_S", "_A", "_V", "_UNKN", "_ADJ", "_ADV", "_INTJ", "_NOUN",
"_PROPN", "_VERB", "_ADP", "_AUX", "_CCONJ", "_DET", "_NUM",
"_PART", "_PRON", "_SCONJ", "_SUM", "_X"
]
tmpPOS = []
if not w2v:
return
for word in vocabulary:
for pos in tmpPOS:
if (word + pos) in w2v.vocab:
W[vocabulary[word]] = w2v[word + pos]
break
for i, pos in enumerate(arrayPOS):
if (word + pos) in w2v.vocab:
tmpPOS.append(pos)
W[vocabulary[word]] = w2v[word + pos]
del arrayPOS[i]
break
net = Sequential(name)
net.setAttr("timestamp", int(time.time()))
net.append(
Embedder(vocabulary,
sentlength,
embsize,
wscale=wscale,
onVocabulary=onVocabulary,
learnable=True,
name="embedder"))
net.append(Reshape((-1, 1, sentlength, embsize)))
branchNum = len(branches)
net.append(Replicate(times=branchNum))
par = Parallel()
for branchFilterSize in branches:
par.append(
buildBranch(branchFilterSize, sentlength, branchMaps, embsize))
net.append(par)
net.append(Concat(axis=1))
net.append(Activation(relu))
net.append(Dropout(p=0.5))
net.append(Linear(branchNum * branchMaps, dim))
return net
def residBlock(inmaps, hmaps, stride, blockname, convShortcut, actInplace,
bnInplace, initscheme):
block = Sequential()
branch = Sequential()
branch.extend(
residMiniBlock(inmaps, hmaps, 1, stride, 0, blockname, "2a", True,
actInplace, bnInplace, initscheme))
branch.extend(
residMiniBlock(hmaps, hmaps, 3, 1, 1, blockname, "2b", True,
actInplace, bnInplace, initscheme))
branch.extend(
residMiniBlock(hmaps, 4 * hmaps, 1, 1, 0, blockname, "2c", False,
actInplace, bnInplace, initscheme))
shortcut = Sequential()
if convShortcut:
shortcut.extend(
residMiniBlock(inmaps, 4 * hmaps, 1, stride, 0, blockname, "1",
False, actInplace, bnInplace, initscheme))
else:
shortcut.append(Identity())
block.append(Replicate(2))
block.append(Parallel().append(branch).append(shortcut))
block.append(Add())
block.append(Activation(relu, inplace=actInplace))
return block
def buildBranch(inmaps=185, inplace=True, num=1, stage=2):
branch = Sequential()
branch.append(
Conv2D(inmaps,
128,
7,
pad=3,
initscheme="none",
name="Mconv1_stage%d_L%d" % (stage, num)))
branch.append(
Activation(relu,
inplace=inplace,
name="Mrelu1_stage%d_L%d" % (stage, num)))
branch.append(
Conv2D(128,
128,
7,
pad=3,
initscheme="none",
name="Mconv2_stage%d_L%d" % (stage, num)))
branch.append(
Activation(relu,
inplace=inplace,
name="Mrelu2_stage%d_L%d" % (stage, num)))
branch.append(
Conv2D(128,
128,
7,
pad=3,
initscheme="none",
name="Mconv3_stage%d_L%d" % (stage, num)))
branch.append(
Activation(relu,
inplace=inplace,
name="Mrelu3_stage%d_L%d" % (stage, num)))
branch.append(
Conv2D(128,
128,
7,
pad=3,
initscheme="none",
name="Mconv4_stage%d_L%d" % (stage, num)))
branch.append(
Activation(relu,
inplace=inplace,
name="Mrelu4_stage%d_L%d" % (stage, num)))
branch.append(
Conv2D(128,
128,
7,
pad=3,
initscheme="none",
name="Mconv5_stage%d_L%d" % (stage, num)))
branch.append(
Activation(relu,
inplace=inplace,
name="Mrelu5_stage%d_L%d" % (stage, num)))
branch.append(
Conv2D(128,
128,
1,
initscheme="none",
name="Mconv6_stage%d_L%d" % (stage, num)))
branch.append(
Activation(relu,
inplace=inplace,
name="Mrelu6_stage%d_L%d" % (stage, num)))
branch.append(
Conv2D(128,
19 * (3 - num),
1,
initscheme="none",
name="Mconv7_stage%d_L%d" % (stage, num)))
return branch
def loadW2L(modelpath, inmaps, nlabels, initscheme=None, name="w2l"):
net = Sequential(name=name)
net.extend(
convBlock(inmaps,
256,
size=11,
stride=2,
pad=5,
dropout=0.2,
initscheme=initscheme,
name="conv1d_0"))
net.extend(
convBlock(256,
256,
size=11,
stride=1,
pad=5,
dropout=0.2,
initscheme=initscheme,
name="conv1d_1"))
net.extend(
convBlock(256,
256,
size=11,
stride=1,
pad=5,
dropout=0.2,
initscheme=initscheme,
name="conv1d_2"))
net.extend(
convBlock(256,
256,
size=11,
stride=1,
pad=5,
dropout=0.2,
initscheme=initscheme,
name="conv1d_3"))
net.extend(
convBlock(256,
384,
size=13,
stride=1,
pad=6,
dropout=0.2,
initscheme=initscheme,
name="conv1d_4"))
net.extend(
convBlock(384,
384,
size=13,
stride=1,
pad=6,
dropout=0.2,
initscheme=initscheme,
name="conv1d_5"))
net.extend(
convBlock(384,
384,
size=13,
stride=1,
pad=6,
dropout=0.2,
initscheme=initscheme,
name="conv1d_6"))
net.extend(
convBlock(384,
512,
size=17,
stride=1,
pad=8,
dropout=0.2,
initscheme=initscheme,
name="conv1d_7"))
net.extend(
convBlock(512,
512,
size=17,
stride=1,
pad=8,
dropout=0.2,
initscheme=initscheme,
name="conv1d_8"))
net.extend(
convBlock(512,
512,
size=17,
stride=1,
pad=8,
dropout=0.2,
initscheme=initscheme,
name="conv1d_9"))
net.extend(
convBlock(512,
640,
size=21,
stride=1,
pad=10,
dropout=0.3,
initscheme=initscheme,
name="conv1d_10"))
net.extend(
convBlock(640,
640,
size=21,
stride=1,
pad=10,
dropout=0.3,
initscheme=initscheme,
name="conv1d_11"))
net.extend(
convBlock(640,
640,
size=21,
stride=1,
pad=10,
dropout=0.3,
initscheme=initscheme,
name="conv1d_12"))
net.extend(
convBlock(640,
768,
size=25,
stride=1,
pad=12,
dropout=0.3,
initscheme=initscheme,
name="conv1d_13"))
net.extend(
convBlock(768,
768,
size=25,
stride=1,
pad=12,
dropout=0.3,
initscheme=initscheme,
name="conv1d_14"))
net.extend(
convBlock(768,
768,
size=25,
stride=1,
pad=12,
dropout=0.3,
initscheme=initscheme,
name="conv1d_15"))
net.extend(
convBlock(768,
896,
size=29,
stride=1,
pad=28,
dropout=0.4,
initscheme=initscheme,
dilation=2,
name="conv1d_16"))
net.extend(
convBlock(896,
1024,
size=1,
stride=1,
pad=0,
dropout=0.4,
initscheme=initscheme,
name="conv1d_17"))
net.extend(
convBlock(1024,
nlabels,
size=1,
stride=1,
pad=0,
dropout=0.0,
initscheme=initscheme,
bnAct=False,
name="conv1d_18"))
if modelpath is not None:
net.load(modelpath)
return net
def convBlock(inmaps,
outmaps,
size,
stride,
pad,
dropout,
initscheme,
dilation=1,
bnAct=True,
name=None):
block = Sequential()
if pad > 0:
block.append(Pad1D(pad, mode=PadMode.reflect))
block.append(
Conv1D(inmaps,
outmaps,
size=size,
stride=stride,
pad=0,
dilation=dilation,
useBias=True,
initscheme=initscheme,
name="%s_conv" % name))
if bnAct:
block.append(BatchNorm1D(outmaps, epsilon=0.001, name="%s_bn" % name))
block.append(Activation(clip, args=(0.0, 20.0)))
if dropout > 0.0:
block.append(Dropout(p=dropout))
return block
def loadVGG(modelpath, layers, poolmode="max", initscheme="none", withLinear=True, actInplace=False, name=None):
if poolmode == "avg":
pool = AvgPool2D
elif poolmode == "max":
pool = MaxPool2D
else:
raise ValueError("Unsupported pool mode")
if layers not in {"11", "16", "19"}:
raise ValueError("Unsupported VGG layers mode")
if name is None and layers == "11":
name = "VGG_ILSVRC_11_layers"
elif name is None and layers == "16":
name = "VGG_ILSVRC_16_layers"
elif name is None and layers == "19":
name = "VGG_ILSVRC_19_layers"
layers = int(layers)
net = Sequential(name=name)
net.append(Conv2D(3, 64, 3, pad=1, initscheme=initscheme, name="conv1_1"))
net.append(Activation(relu, inplace=actInplace, name="relu1_1"))
if layers > 11:
net.append(Conv2D(64, 64, 3, pad=1, initscheme=initscheme, name="conv1_2"))
net.append(Activation(relu, inplace=actInplace, name="relu1_2"))
net.append(pool(2, 2, name="pool1"))
net.append(Conv2D(64, 128, 3, pad=1, initscheme=initscheme, name="conv2_1"))
net.append(Activation(relu, inplace=actInplace, name="relu2_1"))
if layers > 11:
net.append(Conv2D(128, 128, 3, pad=1, initscheme=initscheme, name="conv2_2"))
net.append(Activation(relu, inplace=actInplace, name="relu2_2"))
net.append(pool(2, 2, name="pool2"))
net.append(Conv2D(128, 256, 3, pad=1, initscheme=initscheme, name="conv3_1"))
net.append(Activation(relu, inplace=actInplace, name="relu3_1"))
net.append(Conv2D(256, 256, 3, pad=1, initscheme=initscheme, name="conv3_2"))
net.append(Activation(relu, inplace=actInplace, name="relu3_2"))
if layers > 11:
net.append(Conv2D(256, 256, 3, pad=1, initscheme=initscheme, name="conv3_3"))
net.append(Activation(relu, inplace=actInplace, name="relu3_3"))
if layers > 16:
net.append(Conv2D(256, 256, 3, pad=1, initscheme=initscheme, name="conv3_4"))
net.append(Activation(relu, inplace=actInplace, name="relu3_4"))
net.append(pool(2, 2, name="pool3"))
net.append(Conv2D(256, 512, 3, pad=1, initscheme=initscheme, name="conv4_1"))
net.append(Activation(relu, inplace=actInplace, name="relu4_1"))
net.append(Conv2D(512, 512, 3, pad=1, initscheme=initscheme, name="conv4_2"))
net.append(Activation(relu, inplace=actInplace, name="relu4_2"))
if layers > 11:
net.append(Conv2D(512, 512, 3, pad=1, initscheme=initscheme, name="conv4_3"))
net.append(Activation(relu, inplace=actInplace, name="relu4_3"))
if layers > 16:
net.append(Conv2D(512, 512, 3, pad=1, initscheme=initscheme, name="conv4_4"))
net.append(Activation(relu, inplace=actInplace, name="relu4_4"))
net.append(pool(2, 2, name="pool4"))
net.append(Conv2D(512, 512, 3, pad=1, initscheme=initscheme, name="conv5_1"))
net.append(Activation(relu, inplace=actInplace, name="relu5_1"))
net.append(Conv2D(512, 512, 3, pad=1, initscheme=initscheme, name="conv5_2"))
net.append(Activation(relu, inplace=actInplace, name="relu5_2"))
if layers > 11:
net.append(Conv2D(512, 512, 3, pad=1, initscheme=initscheme, name="conv5_3"))
net.append(Activation(relu, inplace=actInplace, name="relu5_3"))
if layers > 16:
net.append(Conv2D(512, 512, 3, pad=1, initscheme=initscheme, name="conv5_4"))
net.append(Activation(relu, inplace=actInplace, name="relu5_4"))
net.append(pool(2, 2, name="pool5"))
if withLinear:
net.append(Flatten())
insize = int(np.prod(net.dataShapeFrom((1, 3, 224, 224))))
net.append(Linear(insize, 4096, initscheme=initscheme, name="fc6"))
net.append(Activation(relu, inplace=actInplace, name="relu6"))
net.append(Linear(4096, 4096, initscheme=initscheme, name="fc7"))
net.append(Activation(relu, inplace=actInplace, name="relu7"))
net.append(Linear(4096, 1000, initscheme=initscheme, name="fc8"))
net.append(SoftMax())
if modelpath is not None:
net.load(modelpath)
return net
def loadMiniYolo(modelpath, numOutput, actInplace=False, initscheme="none"):
net = Sequential(name="YOLONet")
block0 = block(idx=["1"],
inmaps=[3],
outmaps=[64],
sizeconv=[7],
strideconv=[2],
initscheme=initscheme,
actInPlace=actInplace)
net.extend(block0)
block1 = block(idx=["2"],
inmaps=[64],
outmaps=[192],
sizeconv=[3],
strideconv=[1],
initscheme=initscheme,
actInPlace=actInplace)
net.extend(block1)
block2 = block(idx=["3", "4", "5", "6"],
inmaps=[192, 128, 256, 256],
outmaps=[128, 256, 256, 512],
sizeconv=[1, 3, 1, 3],
strideconv=[1, 1, 1, 1],
initscheme=initscheme,
actInPlace=actInplace)
net.extend(block2)
block3 = block(
idx=["7", "8", "9", "10", "11", "12", "13", "14", "15", "16"],
inmaps=[512, 256, 512, 256, 512, 256, 512, 256, 512, 512],
outmaps=[256, 512, 256, 512, 256, 512, 256, 512, 512, 1024],
sizeconv=[1, 3, 1, 3, 1, 3, 1, 3, 1, 3],
strideconv=[1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
initscheme=initscheme,
actInPlace=actInplace)
net.extend(block3)
block4 = block(idx=["17", "18", "19", "20", "21", "22", "23", "24"],
inmaps=[1024, 512, 1024, 512, 1024, 1024, 1024, 1024],
outmaps=[512, 1024, 512, 1024, 1024, 1024, 1024, 1024],
sizeconv=[1, 3, 1, 3, 3, 3, 3, 3],
strideconv=[1, 1, 1, 1, 1, 2, 1, 1],
initscheme=initscheme,
actInPlace=actInplace,
addMaxpool=False)
net.extend(block4)
net.append(Flatten())
insize = int(np.prod(net.dataShapeFrom((1, 3, 448, 448))))
net.append(Linear(insize, 512, initscheme=initscheme, name="fc25"))
net.append(Activation(relu, inplace=actInplace, name="fc_relu24"))
net.append(Linear(512, 4096, initscheme=initscheme, name="fc26"))
net.append(Activation(relu, inplace=actInplace, name="fc_relu25"))
net.append(Linear(4096, numOutput, initscheme=initscheme, name="fc27"))
net.append(SoftMax())
if modelpath is not None:
net.load(modelpath)
return net
def loadInceptionV3(modelpath,
actInplace=False,
bnInplace=False,
initscheme="none",
name="Inception-7-0001"):
net = Sequential(name=name)
net.append(
Conv2D(3,
32,
3,
stride=2,
useBias=False,
initscheme=initscheme,
name="conv_conv2d"))
net.append(BatchNorm2D(32, name="conv_batchnorm"))
net.append(Activation(relu, inplace=actInplace, name="conv_relu"))
net.append(
Conv2D(32,
32,
3,
useBias=False,
initscheme=initscheme,
name="conv_1_conv2d"))
net.append(BatchNorm2D(32, name="conv_1_batchnorm"))
net.append(Activation(relu, inplace=actInplace, name="conv_1_relu"))
net.append(
Conv2D(32,
64,
3,
pad=1,
useBias=False,
initscheme=initscheme,
name="conv_2_conv2d"))
net.append(BatchNorm2D(64, name="conv_2_batchnorm"))
net.append(Activation(relu, inplace=actInplace, name="conv_2_relu"))
net.append(MaxPool2D(3, 2, name="pool"))
net.append(
Conv2D(64,
80,
1,
useBias=False,
initscheme=initscheme,
name="conv_3_conv2d"))
net.append(BatchNorm2D(80, name="conv_3_batchnorm"))
net.append(Activation(relu, inplace=actInplace, name="conv_3_relu"))
net.append(
Conv2D(80,
192,
3,
useBias=False,
initscheme=initscheme,
name="conv_4_conv2d"))
net.append(BatchNorm2D(192, name="conv_4_batchnorm"))
net.append(Activation(relu, inplace=actInplace, name="conv_4_relu"))
net.append(MaxPool2D(3, 2, name="pool1"))
act, bn = actInplace, bnInplace
net.extend(
bnBlock(192, [64], [48, 64], [64, 96, 96], [32], "mixed", act, bn,
initscheme, 5, 2, "v3"))
net.extend(
bnBlock(256, [64], [48, 64], [64, 96, 96], [64], "mixed_1", act, bn,
initscheme, 5, 2, "v3"))
net.extend(
bnBlock(288, [64], [48, 64], [64, 96, 96], [64], "mixed_2", act, bn,
initscheme, 5, 2, "v3"))
net.extend(
bnShrinkBlock(288, [384], [64, 96, 96], "mixed_3", act, bn, initscheme,
False, 0, "v3"))
net.extend(
factorBlock(768, [192], [128, 128, 192], [128, 128, 128, 128, 192],
[192], "mixed_4", act, bn, initscheme))
net.extend(
factorBlock(768, [192], [160, 160, 192], [160, 160, 160, 160, 192],
[192], "mixed_5", act, bn, initscheme))
net.extend(
factorBlock(768, [192], [160, 160, 192], [160, 160, 160, 160, 192],
[192], "mixed_6", act, bn, initscheme))
net.extend(
factorBlock(768, [192], [192, 192, 192], [192, 192, 192, 192, 192],
[192], "mixed_7", act, bn, initscheme))
net.extend(
v3ShrinkBlock(768, [192, 320], [192, 192, 192, 192], "mixed_8", act,
bn, initscheme))
net.extend(
expandBlock(1280, [320], [384, 384, 384], [448, 384, 384, 384], [192],
"mixed_9",
act,
bn,
initscheme,
pool="avg"))
net.extend(
expandBlock(2048, [320], [384, 384, 384], [448, 384, 384, 384], [192],
"mixed_10",
act,
bn,
initscheme,
pool="max"))
net.append(AvgPool2D(8, 1, name="global_pool"))
net.append(Flatten(name="flatten"))
net.append(Linear(2048, 1008, name="fc1"))
net.append(SoftMax(name="softmax"))
if modelpath is not None:
net.load(modelpath, assumeUniqueNames=True)
return net
def trainSimpleTest(optCls, dtype, *args, **kwargs):
from PuzzleLib.Containers.Sequential import Sequential
from PuzzleLib.Modules.Linear import Linear
from PuzzleLib.Modules.Activation import Activation, relu
from PuzzleLib.Modules.Cast import Cast
from PuzzleLib.Cost.MSE import MSE
seq = Sequential()
seq.append(Linear(128, 64, useBias=False))
seq.append(Activation(relu))
seq.append(Linear(64, 32, useBias=False))
seq.append(Activation(relu))
seq.append(Linear(32, 16))
seq.calcMode(dtype)
seq.append(Cast(intype=dtype, outtype=np.float32))
optimizer = optCls(*args, **kwargs)
optimizer.setupOn(seq, useGlobalState=True)
mse = MSE()
data = gpuarray.to_gpu(np.random.randn(16, 128).astype(dtype))
target = gpuarray.to_gpu(np.random.randn(16, 16).astype(np.float32))
for i in range(200):
error, grad = mse(seq(data), target)
optimizer.zeroGradParams()
seq.backward(grad)
optimizer.update()
if (i + 1) % 5 == 0:
print("Iteration #%d error: %s" % (i + 1, error))
def loadInceptionBN(modelpath,
actInplace=False,
bnInplace=False,
initscheme="none",
name="Inception-BN-0126"):
net = Sequential(name=name)
net.append(
Conv2D(3,
64,
7,
stride=2,
pad=3,
useBias=False,
initscheme=initscheme,
name="conv_1"))
net.append(BatchNorm2D(64, inplace=bnInplace, name="bn_1"))
net.append(Activation(relu, inplace=actInplace, name="relu_1"))
net.append(MaxPool2D(3, 2, pad=1, name="pool_1"))
net.append(
Conv2D(64,
64,
1,
useBias=False,
initscheme=initscheme,
name="conv_2_red"))
net.append(BatchNorm2D(64, inplace=bnInplace, name="bn_2_red"))
net.append(Activation(relu, inplace=actInplace, name="relu_2_red"))
net.append(
Conv2D(64,
192,
3,
pad=1,
useBias=False,
initscheme=initscheme,
name="conv_2"))
net.append(BatchNorm2D(192, inplace=bnInplace, name="bn_2"))
net.append(Activation(relu, inplace=actInplace, name="relu_2"))
net.append(MaxPool2D(3, 2, pad=1, name="pool_2"))
act, bn = actInplace, bnInplace
net.extend(
bnBlock(192, [64], [64, 64], [64, 96, 96], [32],
act=act,
bn=bn,
scheme=initscheme,
name="3a"))
net.extend(
bnBlock(256, [64], [64, 96], [64, 96, 96], [64],
act=act,
bn=bn,
scheme=initscheme,
name="3b"))
net.extend(
bnShrinkBlock(320, [128, 160], [64, 96, 96],
bn=bn,
act=act,
scheme=initscheme,
name="3c"))
net.extend(
bnBlock(576, [224], [64, 96], [96, 128, 128], [128],
act=act,
bn=bn,
scheme=initscheme,
name="4a"))
net.extend(
bnBlock(576, [192], [96, 128], [96, 128, 128], [128],
act=act,
bn=bn,
scheme=initscheme,
name="4b"))
net.extend(
bnBlock(576, [160], [128, 160], [128, 160, 160], [128],
act=act,
bn=bn,
scheme=initscheme,
name="4c"))
net.extend(
bnBlock(608, [96], [128, 192], [160, 192, 192], [128],
act=act,
bn=bn,
scheme=initscheme,
name="4d"))
net.extend(
bnShrinkBlock(608, [128, 192], [192, 256, 256],
act=act,
bn=bn,
scheme=initscheme,
name="4e"))
net.extend(
bnBlock(1056, [352], [192, 320], [160, 224, 224], [128],
act=act,
bn=bn,
scheme=initscheme,
name="5a"))
net.extend(
bnBlock(1024, [352], [192, 320], [192, 224, 224], [128],
act=act,
bn=bn,
scheme=initscheme,
name="5b"))
net.append(AvgPool2D(7, 1, name="global_pool"))
net.append(Flatten(name="flatten"))
net.append(Linear(1024, 1000, initscheme=initscheme, name="fc1"))
net.append(SoftMax(name="softmax"))
if modelpath is not None:
net.load(modelpath, assumeUniqueNames=True)
return net
def loadResNet(modelpath,
layers,
actInplace=False,
bnInplace=False,
initscheme="none",
name=None):
if layers == "50":
if name is None:
name = "ResNet-50"
level3names = ["3%s" % alpha for alpha in string.ascii_lowercase[1:4]]
level4names = ["4%s" % alpha for alpha in string.ascii_lowercase[1:6]]
elif layers == "101":
if name is None:
name = "ResNet-101"
level3names = ["3b%s" % num for num in range(1, 4)]
level4names = ["4b%s" % num for num in range(1, 23)]
elif layers == "152":
if name is None:
name = "ResNet-152"
level3names = ["3b%s" % num for num in range(1, 8)]
level4names = ["4b%s" % num for num in range(1, 36)]
else:
raise ValueError("Unsupported ResNet layers mode")
net = Sequential(name=name)
net.append(
Conv2D(3,
64,
7,
stride=2,
pad=3,
name="conv1",
initscheme=initscheme,
useBias=False))
net.append(BatchNorm2D(64, name="bn_conv1", inplace=bnInplace))
net.append(Activation(relu, inplace=actInplace, name="conv1_relu"))
net.append(MaxPool2D(3, 2, name="pool1"))
net.extend(
residBlock(64, 64, 1, "2a", True, actInplace, bnInplace, initscheme))
net.extend(
residBlock(256, 64, 1, "2b", False, actInplace, bnInplace, initscheme))
net.extend(
residBlock(256, 64, 1, "2c", False, actInplace, bnInplace, initscheme))
net.extend(
residBlock(256, 128, 2, "3a", True, actInplace, bnInplace, initscheme))
for name in level3names:
net.extend(
residBlock(512, 128, 1, name, False, actInplace, bnInplace,
initscheme))
net.extend(
residBlock(512, 256, 2, "4a", True, actInplace, bnInplace, initscheme))
for name in level4names:
net.extend(
residBlock(1024, 256, 1, name, False, actInplace, bnInplace,
initscheme))
net.extend(
residBlock(1024, 512, 2, "5a", True, actInplace, bnInplace,
initscheme))
net.extend(
residBlock(2048, 512, 1, "5b", False, actInplace, bnInplace,
initscheme))
net.extend(
residBlock(2048, 512, 1, "5c", False, actInplace, bnInplace,
initscheme))
net.append(AvgPool2D(7, 1))
net.append(Flatten())
net.append(Linear(2048, 1000, initscheme=initscheme, name="fc1000"))
net.append(SoftMax())
if modelpath is not None:
net.load(modelpath, assumeUniqueNames=True)
return net
def rnnTest(): batchsize, inmaps, inh, inw = 4, 2, 3, 3 outmaps, hsize = 4, 1 seq = Sequential(name="rnn") seq.append(Conv2D(inmaps, outmaps, 3, pad=1)) seq.append(Activation(relu)) seq.append(Reshape(shape=(batchsize, outmaps, inh * inw))) seq.append(SwapAxes(0, 1)) seq.append(RNN(inh * inw, hsize, layers=2, direction="bi", mode="tanh", getSequences=True, hintBatchSize=batchsize)) seq.append(SwapAxes(0, 1)) data = gpuarray.to_gpu(np.random.randn(batchsize, inmaps, inh, inw).astype(np.float32)) engine = buildRTEngine(seq, data.shape, savepath="../TestData", dtype=DataType.float32) outdata = seq(data) enginedata = engine(data) assert np.allclose(outdata.get(), enginedata.get())