def dataflow(name='davis', scale=1, split='val'):
if name == 'davis':
ds = Davis('/data/zubin/videos/davis', name=split, num_frames=1, shuffle=False)
elif name == 'kinetics':
ds = Kinetics('/data/public/rw/datasets/videos/kinetics', num_frames=1, skips=[0], shuffle=False)
else:
raise Exception('not support dataset %s' % name)
if name != 'davis':
ds = df.MapData(ds, lambda dp: [dp[0], dp[1], dp[1]])
ds = df.MapData(ds, lambda dp: [
dp[0], # index
dp[1], # original
dp[2], # mask
dp[3], # name
])
feature_size = int(256 * scale)
size = (feature_size, feature_size)
ds = df.MapDataComponent(ds, ImageProcess.resize(small_axis=feature_size), index=1)
ds = df.MapDataComponent(ds, lambda images: cv2.resize(images[0], size), index=2)
ds = df.MapData(ds, lambda dp: [
dp[0], # index
dp[1][0], # original small axis 256 x scale
cv2.cvtColor(cv2.resize(dp[1][0], size), cv2.COLOR_BGR2GRAY).reshape((size[0], size[1], 1)), # gray (256xscale)x(256xscale)x1
dp[2], # annotated mask 256xscale x 256xscale
dp[3], # name
])
ds = df.MultiProcessPrefetchData(ds, nr_prefetch=32, nr_proc=1)
return ds
def test_davis_tensorpack_dataflow():
ds = Davis('/data/public/rw/datasets/videos/davis/trainval', num_frames=4)
ds = df.MapDataComponent(
ds,
lambda images: [cv2.resize(image, (256, 256)) for image in images],
index=1)
ds = df.MapDataComponent(
ds,
lambda images: [cv2.resize(image, (256, 256)) for image in images],
index=2)
ds = df.MapDataComponent(ds,
lambda images: np.stack(images, axis=0),
index=1)
ds = df.MapDataComponent(ds,
lambda images: np.stack(images, axis=0),
index=2)
ds = df.BatchData(ds, 6)
ds.reset_state()
generator = ds.get_data()
for _ in range(10):
_, images, annotations = next(generator)
assert images.shape == (6, 4, 256, 256, 3)
assert annotations.shape == (6, 4, 256, 256, 3)
def __init__(self,
mode,
batch_size=256,
shuffle=False,
num_workers=25,
cache=50000,
collate_fn=default_collate,
drop_last=False,
cuda=False):
# enumerate standard imagenet augmentors
imagenet_augmentors = fbresnet_augmentor(mode == 'train')
# load the lmdb if we can find it
lmdb_loc = os.path.join(os.environ['IMAGENET'],
'ILSVRC-%s.lmdb' % mode)
ds = td.LMDBData(lmdb_loc, shuffle=False)
ds = td.LocallyShuffleData(ds, cache)
ds = td.PrefetchData(ds, 5000, 1)
ds = td.LMDBDataPoint(ds)
ds = td.MapDataComponent(ds,
lambda x: cv2.imdecode(x, cv2.IMREAD_COLOR),
0)
ds = td.AugmentImageComponent(ds, imagenet_augmentors)
ds = td.PrefetchDataZMQ(ds, num_workers)
self.ds = td.BatchData(ds, batch_size)
self.ds.reset_state()
self.batch_size = batch_size
self.num_workers = num_workers
self.cuda = cuda
def get_input_fn(name, batch_size=32):
image_size = 32
is_training = name == 'train'
ds = df.dataset.Cifar10(name, shuffle=is_training)
ds = df.MapDataComponent(
ds,
lambda x: np.pad(x, [(4, 4), (4, 4), (0, 0)], mode='reflect'),
index=0)
augmentors = [
tp.imgaug.RandomCrop((32, 32)),
tp.imgaug.Flip(horiz=True),
#tp.imgaug.MapImage(lambda x: (x - pp_mean)/128.0),
]
if is_training:
ds = df.RepeatedData(ds, -1)
ds = tp.AugmentImageComponent(ds, augmentors)
else:
ds = tp.AugmentImageComponent(ds, [tp.imgaug.CenterCrop((32, 32))])
ds = tp.AugmentImageComponent(ds,
[tp.imgaug.Resize((image_size, image_size))])
ds = df.MapData(ds, tuple) # for tensorflow.data.dataset
ds.reset_state()
def input_fn():
with tf.name_scope('dataset'):
dataset = tf.data.Dataset.from_generator(
ds.get_data,
output_types=(tf.float32, tf.int64),
output_shapes=(tf.TensorShape([image_size, image_size, 3]),
tf.TensorShape([]))).batch(batch_size)
return dataset
return input_fn
def __init__(self,
mode,
batch_size=256,
shuffle=False,
num_workers=25,
cache=50000,
collate_fn=default_collate,
remainder=False,
cuda=False,
transform=None):
# enumerate standard imagenet augmentors
#imagenet_augmentors = fbresnet_augmentor(mode == 'train')
imagenet_augmentors = [ImgAugTVCompose(transform)]
# load the lmdb if we can find it
lmdb_loc = os.path.join(os.environ['IMAGENET'],
'ILSVRC-%s.lmdb' % mode)
ds = td.LMDBData(lmdb_loc, shuffle=False)
if mode == 'train':
ds = td.LocallyShuffleData(ds, cache)
ds = td.PrefetchData(ds, 5000, 1)
ds = td.LMDBDataPoint(ds)
#ds = td.MapDataComponent(ds, lambda x: cv2.imdecode(x, cv2.IMREAD_COLOR), 0)
ds = td.MapDataComponent(
ds, lambda x: np.asarray(Image.open(io.BytesIO(x)).convert('RGB')),
0)
ds = td.AugmentImageComponent(ds, imagenet_augmentors)
ds = td.PrefetchDataZMQ(ds, num_workers)
self.ds = td.BatchData(ds, batch_size, remainder=remainder)
self.ds.reset_state()
self.batch_size = batch_size
self.num_workers = num_workers
self.cuda = cuda
def dataflow(name='davis', scale=1):
if name == 'davis':
ds = Davis('/data/public/rw/datasets/videos/davis/trainval',
num_frames=1,
shuffle=False)
elif name == 'kinetics':
ds = Kinetics('/data/public/rw/datasets/videos/kinetics',
num_frames=1,
skips=[0],
shuffle=False)
else:
raise Exception('not support dataset %s' % name)
if name != 'davis':
ds = df.MapData(ds, lambda dp: [dp[0], dp[1], dp[1]])
ds = df.MapData(
ds,
lambda dp: [
dp[0], # index
dp[1], # original
dp[2], # mask
])
size = (256 * scale, 256 * scale)
ds = df.MapDataComponent(ds,
ImageProcess.resize(small_axis=256 * scale),
index=1)
ds = df.MapDataComponent(ds,
lambda images: cv2.resize(images[0], size),
index=2)
ds = df.MapData(
ds,
lambda dp: [
dp[0], # index
dp[1][0], # original
cv2.cvtColor(cv2.resize(dp[1][0], size), cv2.COLOR_BGR2GRAY).
reshape((size[0], size[1], 1)), # gray
dp[2], # mask
])
ds = df.MultiProcessPrefetchData(ds, nr_prefetch=32, nr_proc=1)
return ds
def test_kinetics_tensorpack_dataflow():
ds = Kinetics('/data/public/rw/datasets/videos/kinetics',
num_frames=4,
skips=[0, 4, 4, 8])
ds = df.MapDataComponent(
ds,
lambda images: [cv2.resize(image, (256, 256)) for image in images],
index=1)
ds = df.MapDataComponent(ds,
lambda images: np.stack(images, axis=0),
index=1)
ds = df.BatchData(ds, 6)
ds.reset_state()
generator = ds.get_data()
for _ in range(10):
_, images = next(generator)
assert images.shape == (6, 4, 256, 256, 3)
def create_dataflow(data_dir: Path,
kind: str,
batch_size: int,
shuffle: bool = True) -> td.DataFlow:
path = data_dir / "{}.mdb".format(kind)
ds = td.LMDBData(str(path), shuffle=shuffle)
ds = td.MapData(ds, _decode_data)
ds = td.BatchData(ds, batch_size, remainder=False)
ds = td.MapDataComponent(ds, _squeeze_last, index=1)
return ds
def dataflow(name='davis', scale=1):
"""Compute graph to retrieve index, grayscale index, annotation."""
cfg = Config.get_instance()
# get test index one at a time
if name == 'davis':
data_dirpath = cfg['data_dir']['davis']
data = Davis(data_dirpath, num_frames=1, shuffle=False)
elif name == 'kinetics':
data_dirpath = cfg['data_dir']['kinetics']
data = Kinetics(data_dirpath, num_frames=1, skips=[0], shuffle=False)
else:
raise Exception('Dataset [%s] not supported.' % name)
# repeat Kinetics index since Davis has image and annotated frames
if name != 'davis':
data = df.MapData(data, lambda dp: [dp[0], dp[1], dp[1]])
data = df.MapData(data, lambda dp: [dp[0], dp[1], dp[2]])
length = 256 * scale
size = (length, length)
# resize frames to 256x256
data = df.MapDataComponent(data,
ImageProcessor.resize(small_axis=length),
index=1)
data = df.MapDataComponent(data,
lambda images: cv2.resize(images[0], size),
index=2)
# get index, original index, gray scale index, annotation mask
data = df.MapData(
data, lambda dp: [
dp[0],
dp[1][0],
cv2.cvtColor(cv2.resize(dp[1][0], size), cv2.COLOR_BGR2GRAY).
reshape((length, length, 1)),
dp[2],
])
data = df.MultiProcessPrefetchData(data, nr_prefetch=32, nr_proc=1)
return data
def __init__(self,
datafile,
batch_size,
num_workers=1,
nviews=12,
reset=True,
augment=False,
filter_classes=None,
filter_views=None,
polarmode='cartesian',
shuffle=True,
filter_ids=None,
label_to0idx=False,
rgb=False,
force_res=0,
autocrop=False,
keep_aspect_ratio=False):
self.filter_classes = filter_classes
self.filter_views = filter_views
self.filter_ids = filter_ids
self.polarmode = polarmode
self.label_to0idx = label_to0idx
self.rgb = rgb
self.force_res = force_res
self.autocrop = autocrop
self.keep_aspect_ratio = keep_aspect_ratio
if not isinstance(datafile, list):
datafile = [datafile]
ds = []
for d in datafile:
ds.append(df.LMDBSerializer.load(d, shuffle=shuffle))
if shuffle:
ds[-1] = df.LocallyShuffleData(ds[-1], 100)
ds[-1] = df.PrefetchData(ds[-1], 20, 1)
ds[-1] = df.MapData(ds[-1], self.load)
if augment:
ds[-1] = df.MapDataComponent(ds[-1], LMDBMultiView._augment, 0)
if (not filter_classes and not filter_ids and num_workers > 1):
# warning: skipping this is slower when filtering datasets
# but epoch counting will be wrong otherwise
ds[-1] = df.PrefetchDataZMQ(ds[-1], num_workers)
ds[-1] = df.BatchData(ds[-1], batch_size)
if reset:
ds[-1].reset_state()
self.ds = ds
def dataflow(centroids, num_reference=3, num_process=16, shuffle=True):
ds = Kinetics('/data/public/rw/datasets/videos/kinetics',
num_frames=num_reference + 1,
skips=[0, 4, 4, 8][:num_reference + 1],
shuffle=shuffle)
ds = df.MapDataComponent(ds, ImageProcess.resize(small_axis=256), index=1)
ds = df.MapDataComponent(ds, ImageProcess.crop(shape=(256, 256)), index=1)
#ds = df.MapDataComponent(ds, lambda images: [cv2.resize(image, (256, 256)) for image in images], index=1)
ds = df.MapData(
ds, lambda dp: [
dp[1][:num_reference],
copy.deepcopy(dp[1][:num_reference]), dp[1][num_reference:],
copy.deepcopy(dp[1][num_reference:])
])
# for images (ref, target)
for idx in [0, 2]:
ds = df.MapDataComponent(
ds,
lambda images: [
cv2.cvtColor(image, cv2.COLOR_BGR2GRAY).reshape(256, 256, 1)
for image in images
],
index=idx)
# for labels (ref, target)
for idx in [1, 3]:
ds = df.MapDataComponent(
ds,
lambda images: [cv2.resize(image, (32, 32)) for image in images],
index=idx)
ds = df.MapDataComponent(ds,
lambda images: [
cv2.cvtColor(np.float32(image / 255.0),
cv2.COLOR_BGR2Lab)[:, :, 1:]
for image in images
],
index=idx)
ds = df.MapDataComponent(
ds,
lambda images: [
np.array([
np.argmin(np.linalg.norm(centroids - v, axis=1))
for v in image.reshape((-1, 2))
]).reshape((32, 32, 1)) for image in images
],
index=idx)
# stack for tensor
ds = df.MapData(
ds, lambda dp:
[np.stack(dp[0] + dp[2], axis=0),
np.stack(dp[1] + dp[3], axis=0)])
ds = df.MapData(ds, tuple) # for tensorflow.data.dataset
ds = df.MultiProcessPrefetchData(ds, nr_prefetch=256, nr_proc=num_process)
ds = df.PrefetchDataZMQ(ds, nr_proc=1)
return ds
def dataflow3(num_reference=3, num_sets=1, shuffle=True):
kinetics_path = "/media/engin/63c43c7a-cb63-4c43-b70c-f3cb4d68762a/datasets/kinetics/kinetics700"
ds_list = []
for i in range(num_sets):
ds1 = KineticsClustered(i,
kinetics_path,
num_frames=num_reference + 1,
skips=[0, 4, 4, 4][:num_reference + 1],
shuffle=False)
ds1 = df.RepeatedData(ds1, -1)
ds_list.append(ds1)
# ds2 = KineticsClustered(1, kinetics_path, num_frames=num_reference + 1,
# skips=[0, 4, 4, 4][:num_reference + 1], shuffle=False)
# ds2 = df.RepeatedData(ds2, -1)
ds = df.JoinData(ds_list)
# ds = df.MapData(ds, lambda dp: [ [dp[0], dp[1], dp[2]] ])
ds = df.MapData(
ds, lambda dp: [[dp[i], dp[i + 1], dp[i + 2]]
for i in range(0, num_sets * 3, 3)])
# for idx in [0, 1]:
# ds = df.MapDataComponent(ds, lambda dp: [dp[1][:num_reference], dp[2][:num_reference], dp[1][num_reference:], dp[2][num_reference:]], index=idx)
# # stack for tensor
for idx in range(num_sets):
ds = df.MapDataComponent(
ds,
lambda dp: [np.stack(dp[1], axis=0),
np.stack(dp[2], axis=0)],
index=idx)
ds = df.MapData(ds, tuple) # for tensorflow.data.dataset
# ds = df.BatchData(ds, 2, use_list=True)
#Prepare epochs
# ds = df.RepeatedData(ds, total_num_epoch)
# ds = df.RepeatedData(ds, -1)
return ds
def __init__(self,
mode,
do_aug,
batch_size=256,
shuffle=False,
num_workers=25,
cache=50000,
cuda=False,
out_tensor=True,
data_transforms=None):
# enumerate standard imagenet augmentors
imagenet_augmentors = fbresnet_augmentor(do_aug)
# load the lmdb if we can find it
lmdb_loc = os.path.join(os.environ['IMAGENET'],
'ILSVRC-%s.lmdb' % mode)
ds = td.LMDBSerializer.load(lmdb_loc, shuffle=shuffle)
#ds = td.LMDBData(lmdb_loc, shuffle=False)
#ds = td.LocallyShuffleData(ds, cache)
#ds = td.PrefetchData(ds, 5000, 1)
#ds = td.LMDBDataPoint(ds)
ds = td.MapDataComponent(
ds, lambda x: cv2.imdecode(x, cv2.IMREAD_COLOR)[:, :, ::-1], 0)
ds = td.AugmentImageComponent(ds, imagenet_augmentors)
ds = td.MultiProcessRunnerZMQ(ds, num_workers)
self.ds = td.BatchData(ds, batch_size)
self.ds.reset_state()
self.batch_size = batch_size
self.num_workers = num_workers
self.cuda = cuda
self.out_tensor = out_tensor
# data_transforms should be present only when out_tensor=True
# data_transforms typically consists of
# PIL Image transforms, ToTensor(), Normalize():
# normalize = transforms.Compose( [
# transforms.ToTensor(),
# transforms.Normalize(mean=[0.485, 0.456, 0.406],
# std=[0.229, 0.224, 0.225]) ] )
self.data_transforms = data_transforms
print("Loaded '%s'." % lmdb_loc)
else:
logging.basicConfig(level=level, format=log_format, filename=args.log)
logging.info('args: %s', args)
if args.name == 'kinetics':
# get every frame of
ds = Kinetics(kinetics_dirpath, num_frames=1, skips=[0], shuffle=False)
# keep only first frame of each sub-video
# [sub_video_idx, frames[]] -> [first_frame, sub_video_idx]
ds = df.MapData(ds, lambda dp: [dp[1][0], dp[0]])
else:
ds = df.dataset.Cifar10('train', shuffle=False)
logging.info('Downsampling frames to 32x32 resolution')
ds = df.MapDataComponent(ds, lambda image: cv2.resize(image, (32, 32)))
logging.info('Converting RGB to Lab color space')
ds = df.MapDataComponent(ds, lambda image: cv2.cvtColor(np.float32(image / 255.0), cv2.COLOR_RGB2Lab))
ds = df.MapDataComponent(ds, lambda image: image[:, :, 1:])
ds = df.MapDataComponent(ds, lambda image: image.reshape((-1, 2)))
ds = df.RepeatedData(ds, -1)
ds.reset_state()
generator = ds.get_data()
samples = []
for _ in range(args.num_samples):
samples.append(next(generator)[0])
vectors = np.array(samples).reshape((-1, 2))
logging.info('Vectorized images in the shape: %s', vectors.shape)
val_set = data_pipe_3dcnn_block(fmri_data_val,
confounds_val,
label_val,
target_name=target_name,
flag_cnn=Flag_CNN_Model,
block_dura=block_dura,
batch_size=batch_size,
data_type='val',
nr_thread=nr_thread,
buffer_size=buffer_size)
#########################################
def one_hot(label):
return np.eye(len(target_name))[label]
train_gen = dataflow.MapDataComponent(train_gen, one_hot, 1)
val_set = dataflow.MapDataComponent(val_set, one_hot, 1)
###start building models
model_test_GPU_new = KerasModel(
build_model_resnet50,
inputs_desc=[
InputDesc(tf.float32, (None, ) + img_shape, 'images')
],
targets_desc=[
InputDesc(tf.float32, (None, len(target_name)), 'labels')
],
input=train_gen,
trainer=SyncMultiGPUTrainerParameterServer(num_GPU))
model_test_GPU_new.compile(optimizer=opt_use,
def dataflow(centroids, num_refs=3, num_process=16, shuffle=False):
"""
Compute graph to retrieve 3 reference and 1 target frames from Kinetics.
Downsample grayscale frames to 256x256 and colorized frames to 32x32
feature maps in Lab colorspace. Cluster colors in colorized frames.
Returned tensors are of shape (num_refs + 1, 256, 256, 1)
and (num_refs + 1, 32, 32, 1) each. Instead of colorized output,
cluster centroid index is returned.
:return: (grayscale input, cluster indices for colorized output)
"""
config = Config.get_instance()
kinetics_dirpath = config['data_dir']['kinetics']
# get frame and 3 prior reference frames with certain number of skips
data = Kinetics(kinetics_dirpath,
num_frames=num_refs + 1,
skips=[0, 4, 4, 8][:num_refs + 1],
shuffle=shuffle)
# downsample frames to 256x256
data = df.MapDataComponent(data,
ImageProcessor.resize(small_axis=256),
index=1)
data = df.MapDataComponent(data,
ImageProcessor.crop(shape=(256, 256)),
index=1)
# data = df.MapDataComponent(
# data, lambda images: [cv2.resize(image, (256, 256)) for image in images], index=1)
# split frames into 3 references and 1 target frame
# create deep copies of each at odd indices
data = df.MapData(
data, lambda dp: [
dp[1][:num_refs],
copy.deepcopy(dp[1][:num_refs]), dp[1][num_refs:],
copy.deepcopy(dp[1][num_refs:])
])
# decolorize first set of reference and target frames as (256, 256, 1)
for idx in [0, 2]:
data = df.MapDataComponent(
data,
lambda images: [
np.int32(cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)).reshape(
256, 256, 1) for image in images
],
index=idx)
for idx in [1, 3]:
# downsample to 32x32 feature map
data = df.MapDataComponent(
data,
lambda images: [cv2.resize(image, (32, 32)) for image in images],
index=idx)
# discard grayscale L space, keep only 'ab' from Lab color space
# scale from 0-255 to 0-1 for clustering in next step
data = df.MapDataComponent(
data,
lambda images: [
cv2.cvtColor(np.float32(image / 255.0), cv2.COLOR_BGR2Lab)
[:, :, 1:] for image in images
],
index=idx)
# find nearest color cluster index for every pixel in ref and target
data = df.MapDataComponent(
data,
lambda images:
[get_cluster_labels(image, centroids) for image in images],
index=idx)
# combine ref and target frames into (num_refs + 1, dim, dim, 1) tensor
# for both grayscale and colorized feature maps respectively
# generates [input tensor, output tensor]
data = df.MapData(
data, lambda dp:
[np.stack(dp[0] + dp[2], axis=0),
np.stack(dp[1] + dp[3], axis=0)])
# important for tensorflow.data.dataset
# does not do what it is supposed to do
data = df.MapData(data, tuple)
# prefetch 256 datapoints
data = df.MultiProcessPrefetchData(data,
nr_prefetch=256,
nr_proc=num_process)
data = df.PrefetchDataZMQ(data, nr_proc=1)
return data
