def make_movie(model_path, ctx, source='test_idx', fname=None):
from parse import anaglyph, sbs
from parse import split
logging.basicConfig(level=logging.DEBUG)
n_batch = 50
batch_size = 10
start = 0 # 50*24*60/batch_size + 1
scale = (-15, 17)
data_frames = 1
flow_frames = 0
upsample = 4
data_shape = (384, 160)
udata_shape = (data_shape[0] * upsample, data_shape[1] * upsample)
base_shape = (432 * upsample, 180 * upsample)
method = 'multi2'
pos = [(24, 10), (0, 0), (0, 20), (48, 0), (48, 20)]
norm = np.zeros((3, base_shape[1], base_shape[0]), dtype=np.float32)
boarder = 32 * upsample
feather = np.ones((udata_shape[1], udata_shape[0]), dtype=np.float32)
for i in range(udata_shape[1]):
for j in range(udata_shape[0]):
feather[i, j] = min((min(i, j) + 1.0) / boarder, 1.0)
for p in pos:
up = (p[0] * upsample, p[1] * upsample)
norm[:, up[1]:up[1] + udata_shape[1],
up[0]:up[0] + udata_shape[0]] += feather
metric_xy = mx.metric.MAE()
metric_py = mx.metric.MAE()
test_data = data.Mov3dStack('data/lmdb',
data_shape,
batch_size,
scale,
output_depth=False,
data_frames=data_frames,
flow_frames=flow_frames,
test_mode=True,
source=source,
upsample=upsample,
base_shape=base_shape)
sym = make_l1_sym(scale,
'sum',
data_frames,
flow_frames,
False,
method=method,
upsample=upsample)
init = mx.init.Load(model_path, verbose=True)
model = mx.model.FeedForward(ctx=ctx, symbol=sym, initializer=init)
model._init_params(dict(test_data.provide_data + test_data.provide_label))
lcap = cv2.VideoWriter()
pcap_ana = cv2.VideoWriter()
pcap_sbs = cv2.VideoWriter()
ycap_ana = cv2.VideoWriter()
ycap_sbs = cv2.VideoWriter()
lcap.open(fname + '_l.mkv', cv2.VideoWriter_fourcc(*'X264'), 24,
base_shape)
pcap_ana.open(fname + '_ana_p.mkv', cv2.VideoWriter_fourcc(*'X264'), 24,
base_shape)
pcap_sbs.open(fname + '_sbs_p.mkv', cv2.VideoWriter_fourcc(*'X264'), 24,
base_shape)
data_names = [x[0] for x in test_data.provide_data]
model._init_predictor(test_data.provide_data)
data_arrays = [model._pred_exec.arg_dict[name] for name in data_names]
for i in range(n_batch):
X = np.zeros((batch_size, ) + norm.shape, dtype=np.float32)
Y = np.zeros((batch_size, ) + norm.shape, dtype=np.float32)
P = np.zeros((batch_size, ) + norm.shape, dtype=np.float32)
for p in pos:
test_data.seek(start + i)
test_data.fix_p = p
batch = test_data.next()
mx.executor._load_data(batch, data_arrays)
model._pred_exec.forward(is_train=False)
up = (p[0] * upsample, p[1] * upsample)
X[:, :, up[1]:up[1] + udata_shape[1], up[0]:up[0] +
udata_shape[0]] += batch.data[-1].asnumpy() * feather
Y[:, :, up[1]:up[1] + udata_shape[1], up[0]:up[0] +
udata_shape[0]] += batch.label[0].asnumpy() * feather
P[:, :, up[1]:up[1] + udata_shape[1], up[0]:up[0] +
udata_shape[0]] += model._pred_exec.outputs[0].asnumpy(
) * feather
X = np.clip(X / norm, 0, 255)
Y = np.clip(Y / norm, 0, 255)
P = np.clip(P / norm, 0, 255)
metric_py.update([mx.nd.array(Y)], [mx.nd.array(P)])
metric_xy.update([mx.nd.array(Y)], [mx.nd.array(X)])
print i, metric_py.get(), metric_xy.get()
X = X.astype(np.uint8).transpose((0, 2, 3, 1))
Y = Y.astype(np.uint8).transpose((0, 2, 3, 1))
P = P.astype(np.uint8).transpose((0, 2, 3, 1))
for i in range(X.shape[0]):
lcap.write(X[i])
for i in range(Y.shape[0]):
ycap_ana.write(anaglyph(X[i], Y[i]))
ycap_sbs.write(sbs(X[i], Y[i]))
for i in range(P.shape[0]):
pcap_ana.write(anaglyph(X[i], P[i]))
pcap_sbs.write(sbs(X[i], P[i]))
def train(batch_size, prefix):
logging.basicConfig(level=logging.DEBUG)
flog = logging.FileHandler(filename=prefix+datetime.now().strftime('_%Y_%m_%d-%H_%M.log'), mode='w')
flog.setLevel(logging.DEBUG)
logging.getLogger('').addHandler(flog)
scale = (-15, 17)
data_frames = 1
flow_frames = 0
stride = 1
method = 'multi2'
if method == 'direct':
no_left0 = True
right_whiten = True
else:
no_left0 = False
right_whiten = False
tan = False
net = sym.make_l1_sym(scale, 'sum', data_frames, flow_frames, method=method, tan=tan)
train_data = data.Mov3dStack('data/lmdb', (384,160), batch_size, scale, output_depth=False,
data_frames=data_frames, flow_frames=flow_frames, stride=stride, no_left0=no_left0, right_whiten=right_whiten)
test_data = data.Mov3dStack('data/lmdb', (384,160), batch_size, scale, output_depth=False,
data_frames=data_frames, flow_frames=flow_frames,
test_mode=True, stride=stride, no_left0=no_left0, right_whiten=right_whiten)
test_data = mx.io.ResizeIter(test_data, 100)
checkpoint = mx.callback.do_checkpoint(prefix)
def stat(x):
return mx.nd.norm(x)/np.sqrt(x.size)
def std_stat(x):
return [(mx.nd.norm(x)**2-mx.nd.sum(x)**2)/(x.size)]
mon = mx.monitor.Monitor(1, stat, pattern='.*output.*', sort=True)#pred.*output.*|softmax_output|feature_output')
def bilinear(name, arr):
assert name.startswith('deconv_pred')
weight = np.zeros(np.prod(arr.shape[2:]), dtype='float32')
shape = arr.shape
f = np.ceil(shape[3] / 2.)
c = (2 * f - 1 - f % 2) / (2. * f)
for i in range(np.prod(shape[2:])):
x = i % shape[3]
y = (i / shape[3]) % shape[2]
weight[i] = (1 - abs(x / f - c)) * (1 - abs(y / f - c))
weight2 = np.zeros(shape, dtype='float32')
weight = weight.reshape(shape[2:])
for i in range(shape[0]):
weight2[i,i] = weight
arr[:] = weight2.reshape(shape)
vgg16 = data.load_vgg(data_frames, flow_frames, two_stream=False)
init = mx.init.Load(vgg16, mx.init.Uniform(0.01), True)
init = mx.init.Mixed(['deconv_pred.*weight', '.*'], [bilinear, init])
model = mx.model.FeedForward(
ctx = [mx.gpu(0)],
symbol = net,
num_epoch = 100,
learning_rate = 0.002,
lr_scheduler = mx.lr_scheduler.FactorScheduler(20*1000, 0.1),
momentum = 0.9,
wd = 0.0000,
optimizer = 'ccsgd',
initializer = init,
epoch_size = 1000
)
model.fit(
X = train_data,
kvstore = None,
batch_end_callback = [mx.callback.Speedometer(batch_size, 10), mx.callback.log_train_metric(10, auto_reset=True)],
epoch_end_callback = [checkpoint],
#monitor = mon,
eval_metric = mx.metric.MAE(),
eval_data = test_data)
def make_movie(model_path, ctx, source='test_idx', fname=None):
from parse import anaglyph, sbs
from parse import split
logging.basicConfig(level=logging.DEBUG)
n_batch = 50
batch_size = 10
start = 0 # 50*24*60/batch_size + 1
scale = (-15, 17)
data_frames = 1
flow_frames = 0
upsample = 4
data_shape = (384, 160)
udata_shape = (data_shape[0]*upsample, data_shape[1]*upsample)
base_shape = (432*upsample, 180*upsample)
method = 'multi2'
pos = [(24,10), (0,0), (0,20), (48,0), (48,20)]
norm = np.zeros((3, base_shape[1], base_shape[0]), dtype=np.float32)
boarder = 32*upsample
feather = np.ones((udata_shape[1], udata_shape[0]), dtype=np.float32)
for i in range(udata_shape[1]):
for j in range(udata_shape[0]):
feather[i, j] = min((min(i, j)+1.0)/boarder, 1.0)
for p in pos:
up = (p[0]*upsample, p[1]*upsample)
norm[:, up[1]:up[1]+udata_shape[1], up[0]:up[0]+udata_shape[0]] += feather
metric_xy = mx.metric.MAE()
metric_py = mx.metric.MAE()
test_data = data.Mov3dStack('data/lmdb', data_shape, batch_size, scale, output_depth=False,
data_frames=data_frames, flow_frames=flow_frames,
test_mode=True, source=source, upsample=upsample, base_shape=base_shape)
sym = make_l1_sym(scale, 'sum', data_frames, flow_frames, False, method=method, upsample=upsample)
init = mx.init.Load(model_path, verbose=True)
model = mx.model.FeedForward(ctx=ctx, symbol=sym, initializer=init)
model._init_params(dict(test_data.provide_data+test_data.provide_label))
lcap = cv2.VideoWriter()
pcap_ana = cv2.VideoWriter()
pcap_sbs = cv2.VideoWriter()
ycap_ana = cv2.VideoWriter()
ycap_sbs = cv2.VideoWriter()
lcap.open(fname+'_l.mkv', cv2.VideoWriter_fourcc(*'X264'), 24, base_shape)
pcap_ana.open(fname+'_ana_p.mkv', cv2.VideoWriter_fourcc(*'X264'), 24, base_shape)
pcap_sbs.open(fname+'_sbs_p.mkv', cv2.VideoWriter_fourcc(*'X264'), 24, base_shape)
data_names = [x[0] for x in test_data.provide_data]
model._init_predictor(test_data.provide_data)
data_arrays = [model._pred_exec.arg_dict[name] for name in data_names]
for i in range(n_batch):
X = np.zeros((batch_size,)+norm.shape, dtype=np.float32)
Y = np.zeros((batch_size,)+norm.shape, dtype=np.float32)
P = np.zeros((batch_size,)+norm.shape, dtype=np.float32)
for p in pos:
test_data.seek(start+i)
test_data.fix_p = p
batch = test_data.next()
mx.executor._load_data(batch, data_arrays)
model._pred_exec.forward(is_train=False)
up = (p[0]*upsample, p[1]*upsample)
X[:, :, up[1]:up[1]+udata_shape[1], up[0]:up[0]+udata_shape[0]] += batch.data[-1].asnumpy() * feather
Y[:, :, up[1]:up[1]+udata_shape[1], up[0]:up[0]+udata_shape[0]] += batch.label[0].asnumpy() * feather
P[:, :, up[1]:up[1]+udata_shape[1], up[0]:up[0]+udata_shape[0]] += model._pred_exec.outputs[0].asnumpy() * feather
X = np.clip(X/norm, 0, 255)
Y = np.clip(Y/norm, 0, 255)
P = np.clip(P/norm, 0, 255)
metric_py.update([mx.nd.array(Y)], [mx.nd.array(P)])
metric_xy.update([mx.nd.array(Y)], [mx.nd.array(X)])
print i, metric_py.get(), metric_xy.get()
X = X.astype(np.uint8).transpose((0, 2, 3, 1))
Y = Y.astype(np.uint8).transpose((0, 2, 3, 1))
P = P.astype(np.uint8).transpose((0, 2, 3, 1))
for i in range(X.shape[0]):
lcap.write(X[i])
for i in range(Y.shape[0]):
ycap_ana.write(anaglyph(X[i], Y[i]))
ycap_sbs.write(sbs(X[i], Y[i]))
for i in range(P.shape[0]):
pcap_ana.write(anaglyph(X[i], P[i]))
pcap_sbs.write(sbs(X[i], P[i]))
