def predict_flow_stack_list(self, flow_stack_list, score_name, over_sample=True, frame_size=None):
if frame_size is not None:
for i in xrange(len(flow_stack_list)):
flow_stack_list[i] = np.array([cv2.resize(x, frame_size) for x in flow_stack_list[i]])
if over_sample:
tmp = [flow_stack_oversample(stack, (self._sample_shape[2], self._sample_shape[3]))
for stack in flow_stack_list]
os_frame = np.concatenate(tmp, axis=0)
else:
os_frame = np.array(flow_stack_list)
os_frame -= 128
bs = self._sample_shape[0]
feed_data = np.zeros(self._sample_shape)
score_list = []
for offset in xrange(0, os_frame.shape[0], bs):
step = min(bs, os_frame.shape[0] - offset)
feed_data[:step, ...] = os_frame[offset:offset + step, ...]
with self._parrots_session.flow("main") as flow:
flow.set_input('data', feed_data.astype(np.float32, order='C'))
flow.forward()
score_list.append(flow.data(score_name).value().T[:step])
if over_sample:
tmp = np.concatenate(score_list, axis=0)
return tmp.reshape((len(os_frame) / 10, 10, score_list[0].shape[-1]))
else:
return np.concatenate(score_list, axis=0)
def predict_single_flow_stack(self, frames, score_name, over_sample=True):
"""
Predict from a single frame stack
:param frames: (N,H,W)
:param score_name: name of the read-out layer
:param over_sample: data augmentation with oversampling + horizontal flipping
:return:
"""
if over_sample:
os_frame = flow_stack_oversample(
frames, (self._sample_shape[2], self._sample_shape[3]))
else:
os_frame = np.array([
frames,
])
# centering
data = os_frame - 128
self._net.blobs['data'].reshape(*data.shape)
self._net.reshape()
out = self._net.forward(blobs=[
score_name,
], data=data)
return out[score_name].copy()
def predict_single_flow_stack(self,
frame,
score_name,
over_sample=True,
frame_size=None):
if frame_size is not None:
frame = fast_list2arr([cv2.resize(x, frame_size) for x in frame])
else:
frame = fast_list2arr(frame)
if over_sample:
# (4 corner + 1 center) * 2
os_frame = flow_stack_oversample(
frame, (self._sample_shape[2], self._sample_shape[3]))
else:
os_frame = fast_list2arr([frame])
data = os_frame - np.float32(128.0)
self._net.blobs['data'].reshape(*data.shape)
self._net.reshape()
out = self._net.forward(blobs=[
score_name,
], data=data)
return out[score_name].copy()
def predict_flow_stack_list(self, flow_stack_list, score_name, over_sample=True, frame_size=None):
if frame_size is not None:
for i in xrange(len(flow_stack_list)):
flow_stack_list[i] = np.array([cv2.resize(x, frame_size) for x in flow_stack_list[i]])
if over_sample:
tmp = [flow_stack_oversample(stack, (self._sample_shape[2], self._sample_shape[3]))
for stack in flow_stack_list]
os_frame = np.concatenate(tmp, axis=0)
else:
os_frame = np.array(flow_stack_list)
os_frame -= 128
bs = self._sample_shape[0]
feed_data = np.zeros(self._sample_shape)
score_list = []
for offset in xrange(0, os_frame.shape[0], bs):
step = min(bs, os_frame.shape[0] - offset)
feed_data[:step, ...] = os_frame[offset:offset + step, ...]
with self._parrots_session.flow("main") as flow:
flow.set_input('data', feed_data.astype(np.float32, order='C'))
flow.forward()
score_list.append(flow.data(score_name).value().T[:step])
if over_sample:
tmp = np.concatenate(score_list, axis=0)
return tmp.reshape((len(os_frame) / 10, 10, score_list[0].shape[-1]))
else:
return np.concatenate(score_list, axis=0)
def predict_single_flow_rgb_stack_3(self,
flow_frame,
rgb_frame,
score_name,
over_sample=True,
frame_size=None,
multiscale=None,
score_name_1=None,
score_name_2=None):
flow_1 = fast_list2arr([cv2.resize(x, frame_size) for x in flow_frame])
flow_2 = flow_stack_oversample(
flow_1, (self._sample_shape[2], self._sample_shape[3]))
flow_data = flow_2 - np.float32(128.0)
rgb_1 = [cv2.resize(x, frame_size) for x in rgb_frame]
rgb_2 = oversample(rgb_1,
(self._sample_shape[2], self._sample_shape[3]))
# rgb_data1 = fast_list2arr(os_frame_rgb)
# print rgb_data1.shape
def preprocess_1(r):
r = r.transpose(2, 0, 1)
r[0, :, :] = r[0, :, :] - 104
r[1, :, :] = r[1, :, :] - 117
r[2, :, :] = r[2, :, :] - 123
return r
rgb_data = fast_list2arr([preprocess_1(x) for x in rgb_2])
# print flow_data.shape
# print rgb_data.shape
# flow_data = np.reshape(flow_data, (10,-1,224,224))
rgb_data = np.reshape(rgb_data, (10, -1, 224, 224))
# print flow_data.shape
# print rgb_data.shape
# data = np.array([], dtype = rgb_data[0].dtype)
data = np.concatenate((flow_data, rgb_data), axis=1)
# print data.shape
self._net.blobs['data'].reshape(*data.shape)
self._net.reshape()
out = self._net.forward(blobs=[
score_name,
], data=data)
if score_name_1 is not None and score_name_2 is not None:
out_1 = self._net.forward(blobs=[
score_name_1,
], data=data)
out_2 = self._net.forward(blobs=[
score_name_2,
], data=data)
# print "here"
return out[score_name].copy(), out_1[score_name_1].copy(
), out_2[score_name_2].copy()
return out[score_name].copy()
def predict_single_flow_stack(self, frame, score_name, over_sample=True):
if over_sample:
os_frame = flow_stack_oversample(frame, (self._sample_shape[2], self._sample_shape[3]))
else:
os_frame = np.array([frame,])
data = os_frame - 128
self._net.blobs['data'].reshape(*data.shape)
self._net.reshape()
out = self._net.forward(blobs=[score_name,], data=data)
return out[score_name].copy()
def predict_single_flow_stack(self, frame, score_name, over_sample=True):
if over_sample:
os_frame = flow_stack_oversample(
frame, (self._sample_shape[2], self._sample_shape[3]))
else:
os_frame = np.array([
frame,
])
data = os_frame - 128
self._net.blobs['data'].reshape(*data.shape)
self._net.reshape()
out = self._net.forward(blobs=[
score_name,
], data=data)
return out[score_name].copy()
def predict_single_flow_stack(self, frame, score_name, over_sample=True, frame_size=None):
if frame_size is not None:
frame = fast_list2arr([cv2.resize(x, frame_size) for x in frame])
else:
frame = fast_list2arr(frame)
if over_sample:
os_frame = flow_stack_oversample(frame, (self._sample_shape[2], self._sample_shape[3]))
else:
os_frame = fast_list2arr([frame])
data = os_frame - np.float32(128.0)
self._net.blobs['data'].reshape(*data.shape)
self._net.reshape()
out = self._net.forward(blobs=[score_name,], data=data)
return out[score_name].copy()
def predict_single_flow_stack_feature_map(self,
frame,
score_name,
over_sample=False,
frame_size=None,
blobname='conv1/7x7_s2',
dim=30):
if frame_size is not None:
frame = fast_list2arr([cv2.resize(x, frame_size) for x in frame])
else:
frame = fast_list2arr(frame)
print "frame", frame.shape
if over_sample:
os_frame = flow_stack_oversample(
frame, (self._sample_shape[2], self._sample_shape[3]))
else:
os_frame = fast_list2arr([frame])
print "os_frame", os_frame.shape
# (10, 256, 340)
# (10, 10, 224, 224)
data = os_frame - np.float32(128.0)
print data.shape
#self._net.blobs['data'].reshape(*data.shape)
print self._net.blobs['data'].data[0].shape
self._net.blobs['data'].data[...] = data
#self._net.reshape()
out = self._net.forward() #data=data
feat = self._net.blobs[blobname].data[0, :dim]
return feat.copy()