def dataflow(centroids, num_reference=3, num_process=16, shuffle=True):
ds = Kinetics('/data/public/rw/datasets/videos/kinetics',
num_frames=num_reference + 1,
skips=[0, 4, 4, 8][:num_reference + 1],
shuffle=shuffle)
ds = df.MapDataComponent(ds, ImageProcess.resize(small_axis=256), index=1)
ds = df.MapDataComponent(ds, ImageProcess.crop(shape=(256, 256)), index=1)
#ds = df.MapDataComponent(ds, lambda images: [cv2.resize(image, (256, 256)) for image in images], index=1)
ds = df.MapData(
ds, lambda dp: [
dp[1][:num_reference],
copy.deepcopy(dp[1][:num_reference]), dp[1][num_reference:],
copy.deepcopy(dp[1][num_reference:])
])
# for images (ref, target)
for idx in [0, 2]:
ds = df.MapDataComponent(
ds,
lambda images: [
cv2.cvtColor(image, cv2.COLOR_BGR2GRAY).reshape(256, 256, 1)
for image in images
],
index=idx)
# for labels (ref, target)
for idx in [1, 3]:
ds = df.MapDataComponent(
ds,
lambda images: [cv2.resize(image, (32, 32)) for image in images],
index=idx)
ds = df.MapDataComponent(ds,
lambda images: [
cv2.cvtColor(np.float32(image / 255.0),
cv2.COLOR_BGR2Lab)[:, :, 1:]
for image in images
],
index=idx)
ds = df.MapDataComponent(
ds,
lambda images: [
np.array([
np.argmin(np.linalg.norm(centroids - v, axis=1))
for v in image.reshape((-1, 2))
]).reshape((32, 32, 1)) for image in images
],
index=idx)
# stack for tensor
ds = df.MapData(
ds, lambda dp:
[np.stack(dp[0] + dp[2], axis=0),
np.stack(dp[1] + dp[3], axis=0)])
ds = df.MapData(ds, tuple) # for tensorflow.data.dataset
ds = df.MultiProcessPrefetchData(ds, nr_prefetch=256, nr_proc=num_process)
ds = df.PrefetchDataZMQ(ds, nr_proc=1)
return ds
def dataflow(name='davis', scale=1, split='val'):
if name == 'davis':
ds = Davis('/data/zubin/videos/davis', name=split, num_frames=1, shuffle=False)
elif name == 'kinetics':
ds = Kinetics('/data/public/rw/datasets/videos/kinetics', num_frames=1, skips=[0], shuffle=False)
else:
raise Exception('not support dataset %s' % name)
if name != 'davis':
ds = df.MapData(ds, lambda dp: [dp[0], dp[1], dp[1]])
ds = df.MapData(ds, lambda dp: [
dp[0], # index
dp[1], # original
dp[2], # mask
dp[3], # name
])
feature_size = int(256 * scale)
size = (feature_size, feature_size)
ds = df.MapDataComponent(ds, ImageProcess.resize(small_axis=feature_size), index=1)
ds = df.MapDataComponent(ds, lambda images: cv2.resize(images[0], size), index=2)
ds = df.MapData(ds, lambda dp: [
dp[0], # index
dp[1][0], # original small axis 256 x scale
cv2.cvtColor(cv2.resize(dp[1][0], size), cv2.COLOR_BGR2GRAY).reshape((size[0], size[1], 1)), # gray (256xscale)x(256xscale)x1
dp[2], # annotated mask 256xscale x 256xscale
dp[3], # name
])
ds = df.MultiProcessPrefetchData(ds, nr_prefetch=32, nr_proc=1)
return ds
def dataflow(name='davis', scale=1):
if name == 'davis':
ds = Davis('/data/public/rw/datasets/videos/davis/trainval',
num_frames=1,
shuffle=False)
elif name == 'kinetics':
ds = Kinetics('/data/public/rw/datasets/videos/kinetics',
num_frames=1,
skips=[0],
shuffle=False)
else:
raise Exception('not support dataset %s' % name)
if name != 'davis':
ds = df.MapData(ds, lambda dp: [dp[0], dp[1], dp[1]])
ds = df.MapData(
ds,
lambda dp: [
dp[0], # index
dp[1], # original
dp[2], # mask
])
size = (256 * scale, 256 * scale)
ds = df.MapDataComponent(ds,
ImageProcess.resize(small_axis=256 * scale),
index=1)
ds = df.MapDataComponent(ds,
lambda images: cv2.resize(images[0], size),
index=2)
ds = df.MapData(
ds,
lambda dp: [
dp[0], # index
dp[1][0], # original
cv2.cvtColor(cv2.resize(dp[1][0], size), cv2.COLOR_BGR2GRAY).
reshape((size[0], size[1], 1)), # gray
dp[2], # mask
])
ds = df.MultiProcessPrefetchData(ds, nr_prefetch=32, nr_proc=1)
return ds
def dataflow(name='davis', scale=1):
"""Compute graph to retrieve index, grayscale index, annotation."""
cfg = Config.get_instance()
# get test index one at a time
if name == 'davis':
data_dirpath = cfg['data_dir']['davis']
data = Davis(data_dirpath, num_frames=1, shuffle=False)
elif name == 'kinetics':
data_dirpath = cfg['data_dir']['kinetics']
data = Kinetics(data_dirpath, num_frames=1, skips=[0], shuffle=False)
else:
raise Exception('Dataset [%s] not supported.' % name)
# repeat Kinetics index since Davis has image and annotated frames
if name != 'davis':
data = df.MapData(data, lambda dp: [dp[0], dp[1], dp[1]])
data = df.MapData(data, lambda dp: [dp[0], dp[1], dp[2]])
length = 256 * scale
size = (length, length)
# resize frames to 256x256
data = df.MapDataComponent(data,
ImageProcessor.resize(small_axis=length),
index=1)
data = df.MapDataComponent(data,
lambda images: cv2.resize(images[0], size),
index=2)
# get index, original index, gray scale index, annotation mask
data = df.MapData(
data, lambda dp: [
dp[0],
dp[1][0],
cv2.cvtColor(cv2.resize(dp[1][0], size), cv2.COLOR_BGR2GRAY).
reshape((length, length, 1)),
dp[2],
])
data = df.MultiProcessPrefetchData(data, nr_prefetch=32, nr_proc=1)
return data
def dataflow(centroids, num_refs=3, num_process=16, shuffle=False):
"""
Compute graph to retrieve 3 reference and 1 target frames from Kinetics.
Downsample grayscale frames to 256x256 and colorized frames to 32x32
feature maps in Lab colorspace. Cluster colors in colorized frames.
Returned tensors are of shape (num_refs + 1, 256, 256, 1)
and (num_refs + 1, 32, 32, 1) each. Instead of colorized output,
cluster centroid index is returned.
:return: (grayscale input, cluster indices for colorized output)
"""
config = Config.get_instance()
kinetics_dirpath = config['data_dir']['kinetics']
# get frame and 3 prior reference frames with certain number of skips
data = Kinetics(kinetics_dirpath,
num_frames=num_refs + 1,
skips=[0, 4, 4, 8][:num_refs + 1],
shuffle=shuffle)
# downsample frames to 256x256
data = df.MapDataComponent(data,
ImageProcessor.resize(small_axis=256),
index=1)
data = df.MapDataComponent(data,
ImageProcessor.crop(shape=(256, 256)),
index=1)
# data = df.MapDataComponent(
# data, lambda images: [cv2.resize(image, (256, 256)) for image in images], index=1)
# split frames into 3 references and 1 target frame
# create deep copies of each at odd indices
data = df.MapData(
data, lambda dp: [
dp[1][:num_refs],
copy.deepcopy(dp[1][:num_refs]), dp[1][num_refs:],
copy.deepcopy(dp[1][num_refs:])
])
# decolorize first set of reference and target frames as (256, 256, 1)
for idx in [0, 2]:
data = df.MapDataComponent(
data,
lambda images: [
np.int32(cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)).reshape(
256, 256, 1) for image in images
],
index=idx)
for idx in [1, 3]:
# downsample to 32x32 feature map
data = df.MapDataComponent(
data,
lambda images: [cv2.resize(image, (32, 32)) for image in images],
index=idx)
# discard grayscale L space, keep only 'ab' from Lab color space
# scale from 0-255 to 0-1 for clustering in next step
data = df.MapDataComponent(
data,
lambda images: [
cv2.cvtColor(np.float32(image / 255.0), cv2.COLOR_BGR2Lab)
[:, :, 1:] for image in images
],
index=idx)
# find nearest color cluster index for every pixel in ref and target
data = df.MapDataComponent(
data,
lambda images:
[get_cluster_labels(image, centroids) for image in images],
index=idx)
# combine ref and target frames into (num_refs + 1, dim, dim, 1) tensor
# for both grayscale and colorized feature maps respectively
# generates [input tensor, output tensor]
data = df.MapData(
data, lambda dp:
[np.stack(dp[0] + dp[2], axis=0),
np.stack(dp[1] + dp[3], axis=0)])
# important for tensorflow.data.dataset
# does not do what it is supposed to do
data = df.MapData(data, tuple)
# prefetch 256 datapoints
data = df.MultiProcessPrefetchData(data,
nr_prefetch=256,
nr_proc=num_process)
data = df.PrefetchDataZMQ(data, nr_proc=1)
return data