def dataflow(name='davis', scale=1, split='val'):
if name == 'davis':
ds = Davis('/data/zubin/videos/davis', name=split, num_frames=1, shuffle=False)
elif name == 'kinetics':
ds = Kinetics('/data/public/rw/datasets/videos/kinetics', num_frames=1, skips=[0], shuffle=False)
else:
raise Exception('not support dataset %s' % name)
if name != 'davis':
ds = df.MapData(ds, lambda dp: [dp[0], dp[1], dp[1]])
ds = df.MapData(ds, lambda dp: [
dp[0], # index
dp[1], # original
dp[2], # mask
dp[3], # name
])
feature_size = int(256 * scale)
size = (feature_size, feature_size)
ds = df.MapDataComponent(ds, ImageProcess.resize(small_axis=feature_size), index=1)
ds = df.MapDataComponent(ds, lambda images: cv2.resize(images[0], size), index=2)
ds = df.MapData(ds, lambda dp: [
dp[0], # index
dp[1][0], # original small axis 256 x scale
cv2.cvtColor(cv2.resize(dp[1][0], size), cv2.COLOR_BGR2GRAY).reshape((size[0], size[1], 1)), # gray (256xscale)x(256xscale)x1
dp[2], # annotated mask 256xscale x 256xscale
dp[3], # name
])
ds = df.MultiProcessPrefetchData(ds, nr_prefetch=32, nr_proc=1)
return ds
def dataflow2(num_reference=3, shuffle=True):
kinetics_path = "/media/engin/63c43c7a-cb63-4c43-b70c-f3cb4d68762a/datasets/kinetics/kinetics700"
ds = KineticsClustered(kinetics_path,
num_frames=num_reference + 1,
skips=[0, 4, 4, 4][:num_reference + 1],
shuffle=False)
# ds = df.MapDataComponent(ds, lambda images: [cv2.resize(image, (256, 256)) for image in images], index=1)
ds = df.MapData(
ds, lambda dp: [
dp[1][:num_reference], dp[2][:num_reference], dp[1][
num_reference:], dp[2][num_reference:]
])
# # for images (ref, target)
# for idx in [0, 2]:
# ds = df.MapDataComponent(ds, lambda images: [cv2.cvtColor(image, cv2.COLOR_BGR2GRAY).reshape(256, 256, 1) for image in images], index=idx)
# stack for tensor
ds = df.MapData(
ds, lambda dp:
[np.stack(dp[0] + dp[2], axis=0),
np.stack(dp[1] + dp[3], axis=0)])
ds = df.MapData(ds, tuple) # for tensorflow.data.dataset
# ds = df.MultiProcessPrefetchData(ds, nr_prefetch=256, nr_proc=num_process)
# ds = df.PrefetchtraDataZMQ(ds, nr_proc=1)
# ds = df.RepeatedData(ds, total_num_epoch)
# ds = df.RepeatedData(ds, -1)
return ds
def dataflow(centroids, num_reference=3, num_process=16, shuffle=True):
ds = Kinetics('/data/public/rw/datasets/videos/kinetics',
num_frames=num_reference + 1,
skips=[0, 4, 4, 8][:num_reference + 1],
shuffle=shuffle)
ds = df.MapDataComponent(ds, ImageProcess.resize(small_axis=256), index=1)
ds = df.MapDataComponent(ds, ImageProcess.crop(shape=(256, 256)), index=1)
#ds = df.MapDataComponent(ds, lambda images: [cv2.resize(image, (256, 256)) for image in images], index=1)
ds = df.MapData(
ds, lambda dp: [
dp[1][:num_reference],
copy.deepcopy(dp[1][:num_reference]), dp[1][num_reference:],
copy.deepcopy(dp[1][num_reference:])
])
# for images (ref, target)
for idx in [0, 2]:
ds = df.MapDataComponent(
ds,
lambda images: [
cv2.cvtColor(image, cv2.COLOR_BGR2GRAY).reshape(256, 256, 1)
for image in images
],
index=idx)
# for labels (ref, target)
for idx in [1, 3]:
ds = df.MapDataComponent(
ds,
lambda images: [cv2.resize(image, (32, 32)) for image in images],
index=idx)
ds = df.MapDataComponent(ds,
lambda images: [
cv2.cvtColor(np.float32(image / 255.0),
cv2.COLOR_BGR2Lab)[:, :, 1:]
for image in images
],
index=idx)
ds = df.MapDataComponent(
ds,
lambda images: [
np.array([
np.argmin(np.linalg.norm(centroids - v, axis=1))
for v in image.reshape((-1, 2))
]).reshape((32, 32, 1)) for image in images
],
index=idx)
# stack for tensor
ds = df.MapData(
ds, lambda dp:
[np.stack(dp[0] + dp[2], axis=0),
np.stack(dp[1] + dp[3], axis=0)])
ds = df.MapData(ds, tuple) # for tensorflow.data.dataset
ds = df.MultiProcessPrefetchData(ds, nr_prefetch=256, nr_proc=num_process)
ds = df.PrefetchDataZMQ(ds, nr_proc=1)
return ds
def get_input_fn(name,
centroids,
batch_size=32,
num_refs=3,
num_process=16,
shuffle=False):
"""Iterate over datapoints in dataflow in mini-batches."""
_ = name
data = dataflow(centroids,
num_refs=num_refs,
num_process=num_process,
shuffle=shuffle)
data = df.MapData(data, tuple)
data.reset_state()
def input_fn():
with tf.name_scope('dataset'):
dataset = tf.data.Dataset.from_generator(
data.get_data,
output_types=(tf.int32, tf.int32),
output_shapes=(tf.TensorShape([num_refs + 1, 256, 256, 1]),
tf.TensorShape([num_refs + 1, 32, 32,
1]))).batch(batch_size)
return dataset
return input_fn
def get_input_fn(name, batch_size=32):
image_size = 32
is_training = name == 'train'
ds = df.dataset.Cifar10(name, shuffle=is_training)
ds = df.MapDataComponent(
ds,
lambda x: np.pad(x, [(4, 4), (4, 4), (0, 0)], mode='reflect'),
index=0)
augmentors = [
tp.imgaug.RandomCrop((32, 32)),
tp.imgaug.Flip(horiz=True),
#tp.imgaug.MapImage(lambda x: (x - pp_mean)/128.0),
]
if is_training:
ds = df.RepeatedData(ds, -1)
ds = tp.AugmentImageComponent(ds, augmentors)
else:
ds = tp.AugmentImageComponent(ds, [tp.imgaug.CenterCrop((32, 32))])
ds = tp.AugmentImageComponent(ds,
[tp.imgaug.Resize((image_size, image_size))])
ds = df.MapData(ds, tuple) # for tensorflow.data.dataset
ds.reset_state()
def input_fn():
with tf.name_scope('dataset'):
dataset = tf.data.Dataset.from_generator(
ds.get_data,
output_types=(tf.float32, tf.int64),
output_shapes=(tf.TensorShape([image_size, image_size, 3]),
tf.TensorShape([]))).batch(batch_size)
return dataset
return input_fn
def __init__(self,
bert_model_name,
captions,
images,
dataset_type,
image_features,
obj_list,
seq_len,
encoding="utf-8"):
self.tokenizer = BertTokenizer.from_pretrained(bert_model_name,
do_lower_case=True)
self.dataset_type = dataset_type
self.imageid2filepath = {}
entries = self._load_entries(images, captions, dataset_type)
preprocess_function = BertPreprocessBatch(
tokenizer=self.tokenizer,
obj_list=obj_list,
seq_len=seq_len,
region_len=36,
image_features=image_features,
imageid2filepath=self.imageid2filepath,
encoding=encoding,
predict_feature=False)
ds = td.MapData(entries, preprocess_function)
self.formatted_entries = list(ds.get_data())
def __init__(self,
corpus_path,
tokenizer,
seq_len,
encoding="utf-8",
predict_feature=False,
batch_size=512,
shuffle=False,
num_workers=25,
cache=10000,
drop_last=False,
cuda=False,
distributed=False,
visualization=False,
span_mask=False,
cond_mask=False,
region_len=36):
if dist.is_available() and distributed:
rank = dist.get_rank()
lmdb_file = os.path.join(
corpus_path, "training_feat_part_" + str(rank) + ".lmdb")
else:
lmdb_file = os.path.join(corpus_path, "training_feat_all.lmdb")
caption_path = os.path.join(corpus_path, "caption_train.json")
print("Loading from %s" % lmdb_file)
os.listdir(corpus_path)
ds = td.LMDBSerializer.load(lmdb_file, shuffle=False)
self.num_dataset = len(ds)
self.cond_mask = cond_mask
preprocess_function = BertPreprocessBatch(
caption_path,
tokenizer,
seq_len,
region_len,
self.num_dataset,
encoding="utf-8",
predict_feature=predict_feature,
span_mask=span_mask,
cond_mask=cond_mask)
# ds = td.LocallyShuffleData(ds, cache)
ds = td.PrefetchData(ds, 5000, 1)
ds = td.MapData(ds, preprocess_function)
# self.ds = td.PrefetchData(ds, 1)
ds = td.PrefetchDataZMQ(ds, num_workers)
self.ds = td.BatchData(ds, batch_size)
# self.ds = ds
self.ds.reset_state()
self.batch_size = batch_size
self.num_workers = num_workers
def __init__(
self,
corpus_path,
tokenizer,
seq_len,
encoding='utf-8',
predict_feature=False,
hard_negative=False,
batch_size=512,
shuffle=False,
num_workers=25,
cache=50000,
drop_last=False,
cuda=False,
distributed=False,
visualization=False,
):
if dist.is_available() and distributed:
# num_replicas = dist.get_world_size()
# assert num_replicas == 8
rank = dist.get_rank()
lmdb_file = '/mnt3/xuesheng/features_lmdb/CC/training_feat_part_' + str(
rank) + '.lmdb'
# if not os.path.exists(lmdb_file):
# lmdb_file = "/srv/share/datasets/conceptual_caption/training_feat_part_" + str(rank) + ".lmdb"
else:
# lmdb_file = "/coc/dataset/conceptual_caption/training_feat_all.lmdb"
# if not os.path.exists(lmdb_file):
lmdb_file = '/mnt3/xuesheng/features_lmdb/CC/training_feat_part_0.lmdb'
caption_path = '/mnt3/xuesheng/features_lmdb/CC/caption_train.json'
print('Loading from %s' % lmdb_file)
ds = td.LMDBSerializer.load(lmdb_file, shuffle=True)
self.num_dataset = len(ds)
preprocess_function = BertPreprocessBatch(
caption_path,
tokenizer,
seq_len,
36,
self.num_dataset,
encoding='utf-8',
predict_feature=predict_feature,
)
# ds = td.LocallyShuffleData(ds, cache)
# ds = td.PrefetchData(ds, 5000, 1)
ds = td.MapData(ds, preprocess_function)
# self.ds = td.PrefetchData(ds, 1)
ds = td.PrefetchDataZMQ(ds, num_workers)
self.ds = td.BatchData(ds, batch_size)
# self.ds = ds
self.ds.reset_state()
self.batch_size = batch_size
self.num_workers = num_workers
def dataflow4(num_reference=3, order=1, shuffle=True):
kinetics_path = "/media/engin/63c43c7a-cb63-4c43-b70c-f3cb4d68762a/datasets/kinetics/kinetics700"
ds = KineticsClustered(order,
kinetics_path,
num_frames=num_reference + 1,
skips=[0, 4, 4, 4][:num_reference + 1],
shuffle=False)
# # stack for tensor
ds = df.MapData(
ds, lambda dp: [np.stack(dp[1], axis=0),
np.stack(dp[2], axis=0)])
ds = df.MapData(ds, tuple) # for tensorflow.data.dataset
ds = df.RepeatedData(ds, -1)
return ds
def dataflow3(num_reference=3, num_sets=1, shuffle=True):
kinetics_path = "/media/engin/63c43c7a-cb63-4c43-b70c-f3cb4d68762a/datasets/kinetics/kinetics700"
ds_list = []
for i in range(num_sets):
ds1 = KineticsClustered(i,
kinetics_path,
num_frames=num_reference + 1,
skips=[0, 4, 4, 4][:num_reference + 1],
shuffle=False)
ds1 = df.RepeatedData(ds1, -1)
ds_list.append(ds1)
# ds2 = KineticsClustered(1, kinetics_path, num_frames=num_reference + 1,
# skips=[0, 4, 4, 4][:num_reference + 1], shuffle=False)
# ds2 = df.RepeatedData(ds2, -1)
ds = df.JoinData(ds_list)
# ds = df.MapData(ds, lambda dp: [ [dp[0], dp[1], dp[2]] ])
ds = df.MapData(
ds, lambda dp: [[dp[i], dp[i + 1], dp[i + 2]]
for i in range(0, num_sets * 3, 3)])
# for idx in [0, 1]:
# ds = df.MapDataComponent(ds, lambda dp: [dp[1][:num_reference], dp[2][:num_reference], dp[1][num_reference:], dp[2][num_reference:]], index=idx)
# # stack for tensor
for idx in range(num_sets):
ds = df.MapDataComponent(
ds,
lambda dp: [np.stack(dp[1], axis=0),
np.stack(dp[2], axis=0)],
index=idx)
ds = df.MapData(ds, tuple) # for tensorflow.data.dataset
# ds = df.BatchData(ds, 2, use_list=True)
#Prepare epochs
# ds = df.RepeatedData(ds, total_num_epoch)
# ds = df.RepeatedData(ds, -1)
return ds
def dataflow(name='davis', scale=1):
if name == 'davis':
ds = Davis('/data/public/rw/datasets/videos/davis/trainval',
num_frames=1,
shuffle=False)
elif name == 'kinetics':
ds = Kinetics('/data/public/rw/datasets/videos/kinetics',
num_frames=1,
skips=[0],
shuffle=False)
else:
raise Exception('not support dataset %s' % name)
if name != 'davis':
ds = df.MapData(ds, lambda dp: [dp[0], dp[1], dp[1]])
ds = df.MapData(
ds,
lambda dp: [
dp[0], # index
dp[1], # original
dp[2], # mask
])
size = (256 * scale, 256 * scale)
ds = df.MapDataComponent(ds,
ImageProcess.resize(small_axis=256 * scale),
index=1)
ds = df.MapDataComponent(ds,
lambda images: cv2.resize(images[0], size),
index=2)
ds = df.MapData(
ds,
lambda dp: [
dp[0], # index
dp[1][0], # original
cv2.cvtColor(cv2.resize(dp[1][0], size), cv2.COLOR_BGR2GRAY).
reshape((size[0], size[1], 1)), # gray
dp[2], # mask
])
ds = df.MultiProcessPrefetchData(ds, nr_prefetch=32, nr_proc=1)
return ds
def parallel(self, num_threads, buffer_size=200, strict=False):
self.is_parallel = True
ds = self
ds = df.MultiThreadMapData(ds,
nr_thread=num_threads,
map_func=NetworkImages.map_func_download,
buffer_size=buffer_size,
strict=strict)
# ds = df.PrefetchDataZMQ(ds, nr_proc=1) # to reduce GIL contention.
ds = df.MapData(ds, func=NetworkImages.map_func_decode)
return ds
def create_dataflow(data_dir: Path,
kind: str,
batch_size: int,
shuffle: bool = True) -> td.DataFlow:
path = data_dir / "{}.mdb".format(kind)
ds = td.LMDBData(str(path), shuffle=shuffle)
ds = td.MapData(ds, _decode_data)
ds = td.BatchData(ds, batch_size, remainder=False)
ds = td.MapDataComponent(ds, _squeeze_last, index=1)
return ds
def dataflow(name='davis', scale=1):
"""Compute graph to retrieve index, grayscale index, annotation."""
cfg = Config.get_instance()
# get test index one at a time
if name == 'davis':
data_dirpath = cfg['data_dir']['davis']
data = Davis(data_dirpath, num_frames=1, shuffle=False)
elif name == 'kinetics':
data_dirpath = cfg['data_dir']['kinetics']
data = Kinetics(data_dirpath, num_frames=1, skips=[0], shuffle=False)
else:
raise Exception('Dataset [%s] not supported.' % name)
# repeat Kinetics index since Davis has image and annotated frames
if name != 'davis':
data = df.MapData(data, lambda dp: [dp[0], dp[1], dp[1]])
data = df.MapData(data, lambda dp: [dp[0], dp[1], dp[2]])
length = 256 * scale
size = (length, length)
# resize frames to 256x256
data = df.MapDataComponent(data,
ImageProcessor.resize(small_axis=length),
index=1)
data = df.MapDataComponent(data,
lambda images: cv2.resize(images[0], size),
index=2)
# get index, original index, gray scale index, annotation mask
data = df.MapData(
data, lambda dp: [
dp[0],
dp[1][0],
cv2.cvtColor(cv2.resize(dp[1][0], size), cv2.COLOR_BGR2GRAY).
reshape((length, length, 1)),
dp[2],
])
data = df.MultiProcessPrefetchData(data, nr_prefetch=32, nr_proc=1)
return data
def __init__(self,
datafile,
batch_size,
num_workers=1,
nviews=12,
reset=True,
augment=False,
filter_classes=None,
filter_views=None,
polarmode='cartesian',
shuffle=True,
filter_ids=None,
label_to0idx=False,
rgb=False,
force_res=0,
autocrop=False,
keep_aspect_ratio=False):
self.filter_classes = filter_classes
self.filter_views = filter_views
self.filter_ids = filter_ids
self.polarmode = polarmode
self.label_to0idx = label_to0idx
self.rgb = rgb
self.force_res = force_res
self.autocrop = autocrop
self.keep_aspect_ratio = keep_aspect_ratio
if not isinstance(datafile, list):
datafile = [datafile]
ds = []
for d in datafile:
ds.append(df.LMDBSerializer.load(d, shuffle=shuffle))
if shuffle:
ds[-1] = df.LocallyShuffleData(ds[-1], 100)
ds[-1] = df.PrefetchData(ds[-1], 20, 1)
ds[-1] = df.MapData(ds[-1], self.load)
if augment:
ds[-1] = df.MapDataComponent(ds[-1], LMDBMultiView._augment, 0)
if (not filter_classes and not filter_ids and num_workers > 1):
# warning: skipping this is slower when filtering datasets
# but epoch counting will be wrong otherwise
ds[-1] = df.PrefetchDataZMQ(ds[-1], num_workers)
ds[-1] = df.BatchData(ds[-1], batch_size)
if reset:
ds[-1].reset_state()
self.ds = ds
def __init__(
self,
annotations_path,
features_path,
tokenizer,
bert_model,
seq_len,
batch_size=512,
num_workers=25,
cache=10000,
local_rank=-1,
objective=0,
num_locs=5,
add_global_imgfeat=None,
):
if dist.is_available() and local_rank != -1:
rank = dist.get_rank()
lmdb_file = os.path.join(
features_path, "training_feat_part_" + str(rank) + ".lmdb")
else:
lmdb_file = os.path.join(features_path, "training_feat_all.lmdb")
print("Loading from %s" % lmdb_file)
ds = td.LMDBSerializer.load(lmdb_file, shuffle=False)
self.num_dataset = len(ds)
ds = td.LocallyShuffleData(ds, cache)
caption_path = os.path.join(annotations_path, "caption_train.json")
preprocess_function = BertPreprocessBatch(
caption_path,
tokenizer,
bert_model,
seq_len,
36,
self.num_dataset,
objective=objective,
num_locs=num_locs,
)
ds = td.PrefetchData(ds, 5000, 1)
ds = td.MapData(ds, preprocess_function)
ds = td.PrefetchDataZMQ(ds, num_workers)
self.ds = td.BatchData(ds, batch_size)
self.ds.reset_state()
self.batch_size = batch_size
self.num_workers = num_workers
self.add_global_imgfeat = add_global_imgfeat
self.num_locs = num_locs
def __init__(self,
corpus_path,
tokenizer,
seq_len,
encoding="utf-8",
predict_feature=False,
batch_size=512,
shuffle=False,
num_workers=25,
cache=50000,
drop_last=False,
cuda=False,
distributed=False,
visualization=False,
span_mask=False,
cond_mask=False,
region_len=36):
lmdb_file = os.path.join(corpus_path, "validation_all.lmdb")
caption_path = os.path.join(corpus_path, "caption_val.json")
print("Loading from %s" % lmdb_file)
ds = td.LMDBSerializer.load(lmdb_file, shuffle=False)
self.num_dataset = len(ds)
self.cond_mask = cond_mask
preprocess_function = BertPreprocessBatch(
caption_path,
tokenizer,
seq_len,
region_len,
self.num_dataset,
encoding="utf-8",
predict_feature=predict_feature,
visualization=visualization,
span_mask=span_mask,
cond_mask=cond_mask,
)
ds = td.MapData(ds, preprocess_function)
self.ds = td.BatchData(ds, batch_size)
self.ds.reset_state()
self.batch_size = batch_size
self.num_workers = num_workers
def init_dataflow(ctfstar,batch_size):
''' This function creates dataflow that reads and preprocesses data in parallel '''
augm = df.imgaug.AugmentorList([df.imgaug.MeanVarianceNormalize()])
# create partitioned generators, one for each element in a batch
dss0,shape = MicrosGenerator.create_partition(ctfstar,batch_size)
# preprocess input
dss1 = [df.MapData(ds0, lambda dp: [augm.augment(preprocess_micro(dp[0], dp[1], psize, bn)), np.array(dp[0])]) for ds0 in dss0]
# prefetch each generator in a separate process with buffer of 4 images per process
# dss1 = [df.PrefetchDataZMQ(ds1, nr_proc=1, hwm=2) for ds1 in dss1]
dss1 = [df.PrefetchData(ds1, nr_prefetch=4, nr_proc=1) for ds1 in dss1]
# join all dataflows
ds1 = df.RandomMixData(dss1)
# ds1 = df.JoinData(dss1)
ds = df.BatchData(ds1, batch_size)
ds.reset_state()
return ds,shape
def __init__(
self,
corpus_path,
tokenizer,
seq_len,
encoding='utf-8',
predict_feature=False,
batch_size=512,
shuffle=False,
num_workers=25,
cache=50000,
drop_last=False,
cuda=False,
distributed=False,
visualization=False,
):
lmdb_file = '/mnt3/xuesheng/features_lmdb/CC_val/validation_all.lmdb'
if not os.path.exists(lmdb_file):
lmdb_file = '/mnt3/xuesheng/features_lmdb/CC_val/validation_all.lmdb'
caption_path = '/mnt3/xuesheng/features_lmdb/CC_val/caption_val.json'
print('Loading from %s' % lmdb_file)
ds = td.LMDBSerializer.load(lmdb_file, shuffle=False)
self.num_dataset = len(ds)
preprocess_function = BertPreprocessBatch(
caption_path,
tokenizer,
seq_len,
36,
self.num_dataset,
encoding='utf-8',
predict_feature=predict_feature,
visualization=visualization,
)
ds = td.MapData(ds, preprocess_function)
self.ds = td.BatchData(ds, batch_size)
self.ds.reset_state()
self.batch_size = batch_size
self.num_workers = num_workers
def __init__(
self,
corpus_path,
tokenizer,
seq_len,
encoding="utf-8",
predict_feature=False,
batch_size=512,
shuffle=False,
num_workers=25,
cache=50000,
drop_last=False,
cuda=False,
distributed=False,
visualization=False,
):
lmdb_file = "/coc/dataset/conceptual_caption/validation_feat_all.lmdb"
if not os.path.exists(lmdb_file):
lmdb_file = "/coc/pskynet2/jlu347/multi-modal-bert/data/conceptual_caption/validation_feat_all.lmdb"
caption_path = "/coc/pskynet2/jlu347/multi-modal-bert/data/conceptual_caption/caption_val.json"
print("Loading from %s" % lmdb_file)
ds = td.LMDBSerializer.load(lmdb_file, shuffle=False)
self.num_dataset = len(ds)
preprocess_function = BertPreprocessBatch(
caption_path,
tokenizer,
seq_len,
36,
self.num_dataset,
encoding="utf-8",
predict_feature=predict_feature,
visualization=visualization,
)
ds = td.MapData(ds, preprocess_function)
self.ds = td.BatchData(ds, batch_size)
self.ds.reset_state()
self.batch_size = batch_size
self.num_workers = num_workers
def __init__(
self,
corpus_path,
tokenizer,
bert_model,
seq_len,
encoding="utf-8",
visual_target=0,
batch_size=512,
shuffle=False,
num_workers=25,
cache=5000,
drop_last=False,
cuda=False,
objective=0,
visualization=False,
):
lmdb_file = os.path.join(corpus_path, "validation_feat_all.lmdb")
caption_path = os.path.join(corpus_path, "caption_val.json")
print("Loading from %s" % lmdb_file)
ds = td.LMDBSerializer.load(lmdb_file, shuffle=False)
self.num_dataset = len(ds)
preprocess_function = BertPreprocessBatch(
caption_path,
tokenizer,
bert_model,
seq_len,
36,
self.num_dataset,
encoding="utf-8",
visual_target=visual_target,
visualization=visualization,
objective=objective,
)
ds = td.MapData(ds, preprocess_function)
self.ds = td.BatchData(ds, batch_size)
self.ds.reset_state()
self.batch_size = batch_size
self.num_workers = num_workers
def __init__(
self,
annotations_path,
features_path,
tokenizer,
bert_model,
seq_len,
batch_size=512,
num_workers=25,
cache=5000,
objective=0,
num_locs=5,
add_global_imgfeat=True,
visualization=False,
):
lmdb_file = os.path.join(features_path, "validation_feat_all.lmdb")
caption_path = os.path.join(annotations_path, "caption_valid.json")
print("Loading from %s" % lmdb_file)
ds = td.LMDBSerializer.load(lmdb_file, shuffle=False)
self.num_dataset = len(ds)
preprocess_function = BertPreprocessBatch(
caption_path,
tokenizer,
bert_model,
seq_len,
36,
self.num_dataset,
visualization=visualization,
objective=objective,
num_locs=num_locs,
)
ds = td.MapData(ds, preprocess_function)
self.ds = td.BatchData(ds, batch_size)
self.ds.reset_state()
self.batch_size = batch_size
self.num_workers = num_workers
self.add_global_imgfeat = add_global_imgfeat
self.num_locs = num_locs
def lmdb_dataflow(lmdb_path,
batch_size,
input_size,
output_size,
is_training,
test_speed=False,
filter_rate=0):
df = dataflow.LMDBSerializer.load(lmdb_path, shuffle=False)
df = dataflow.MapData(df,
lambda dp: [item for item in dp] + [random.random()])
size = df.size()
print(size)
if is_training:
df = dataflow.LocallyShuffleData(df, buffer_size=2000)
df = dataflow.PrefetchData(df, nr_prefetch=500, nr_proc=1)
df = BatchData(df, batch_size, input_size, output_size)
if is_training:
df = dataflow.PrefetchDataZMQ(df, nr_proc=8)
df = dataflow.RepeatedData(df, -1)
if test_speed:
dataflow.TestDataSpeed(df, size=1000).start()
df.reset_state()
return df, size
df.imgaug.CenterPaste((36, 36)),
df.imgaug.RandomCrop((32, 32)),
df.imgaug.RandomOrderAug([
df.imgaug.RandomApplyAug(
df.imgaug.BrightnessScale((0.8, 1.2), clip=False), 0.5),
df.imgaug.RandomApplyAug(
df.imgaug.Contrast((0.8, 1.2), clip=False), 0.5),
# df.imgaug.RandomApplyAug(df.imgaug.Saturation(0.4, rgb=False), 0.5),
]),
]
augmentors_default = [
df.imgaug.Resize((32, 32)),
df.imgaug.MapImage(lambda x: x.reshape(32, 32, 1))
]
# keep original image at index 1
ds = df.MapData(
ds, lambda datapoint: [datapoint[0], datapoint[0]] + datapoint[1:])
ds = df.AugmentImageComponent(ds,
augmentors_variation + augmentors_default)
ds = df.AugmentImageComponent(ds, augmentors_default, index=1)
ds = df.PrefetchData(ds, nr_prefetch=12, nr_proc=4)
ds = df.PrintData(ds)
ds = df.BatchData(ds, batch_size=32, remainder=False, use_list=True)
ds = df.PrintData(ds)
for minibatch in ds.get_data():
images, originals, labels = minibatch
image, original, label = images[0], originals[0], labels[0]
name = '{:02d}'.format(label)
cv2.namedWindow(name)
cv2.moveWindow(name, 0, 25 + 128 * label)
def __init__(
self,
corpus_path,
tokenizer,
seq_len,
encoding='utf-8',
predict_feature=False,
batch_size=512,
shuffle=False,
num_workers=10,
cache=50000,
drop_last=False,
cuda=False,
):
lmdb_file = '/coc/dataset/conceptual_caption/validation_feat_all.lmdb'
if not os.path.exists(lmdb_file):
lmdb_file = '/coc/pskynet2/jlu347/multi-modal-bert/data/conceptual_caption/validation_feat_all.lmdb'
caption_path = '/coc/pskynet2/jlu347/multi-modal-bert/data/conceptual_caption/caption_val.json'
print('Loading from %s' % lmdb_file)
ds = td.LMDBSerializer.load(lmdb_file, shuffle=False)
self.num_dataset = len(ds)
preprocess_function = BertPreprocessRetrieval(
caption_path,
tokenizer,
seq_len,
36,
1000,
encoding='utf-8',
predict_feature=predict_feature,
)
ds = td.MapData(ds, preprocess_function)
self.ds = td.BatchData(ds, 1)
self.ds.reset_state()
self.batch_size = 1
self.num_workers = num_workers
self._entry = []
self.features_all = np.zeros((1000, 37, 2048), dtype=np.float32)
self.spatials_all = np.zeros((1000, 37, 5), dtype=np.float32)
self.image_mask_all = np.zeros((1000, 37), dtype=np.float32)
self.image_ids = []
# load first 1000 file here.
for i, batch in enumerate(self.ds.get_data()):
if i >= 1000:
break
input_ids, input_mask, segment_ids, is_next, image_feat, image_loc, image_mask, image_id, caption = batch
batch_size = input_ids.shape[0]
g_image_feat = np.sum(image_feat, axis=1) / np.sum(
image_mask, axis=1, keepdims=True)
image_feat = np.concatenate(
[np.expand_dims(g_image_feat, axis=1), image_feat], axis=1)
image_feat = np.array(image_feat, dtype=np.float32)
g_image_loc = np.repeat(np.array([[0, 0, 1, 1, 1]],
dtype=np.float32),
batch_size,
axis=0)
image_loc = np.concatenate(
[np.expand_dims(g_image_loc, axis=1), image_loc], axis=1)
image_loc = np.array(image_loc, dtype=np.float32)
g_image_mask = np.repeat(np.array([[1]]), batch_size, axis=0)
image_mask = np.concatenate([g_image_mask, image_mask], axis=1)
batch = (input_ids, input_mask, segment_ids, image_id, caption)
self._entry.append(batch)
self.features_all[i] = image_feat
self.image_mask_all[i] = np.array(image_mask)
self.spatials_all[i] = image_loc
self.image_ids.append(image_id)
sys.stdout.write('%d/%d\r' % (i, 1000))
sys.stdout.flush()
def __init__(
self,
corpus_path,
tokenizer,
bert_model,
seq_len,
encoding="utf-8",
visual_target=0,
hard_negative=False,
batch_size=512,
shuffle=False,
num_workers=25,
cache=10000,
drop_last=False,
cuda=False,
local_rank=-1,
objective=0,
visualization=False,
):
TRAIN_DATASET_SIZE = 3119449
if dist.is_available() and local_rank != -1:
num_replicas = dist.get_world_size()
rank = dist.get_rank()
lmdb_file = os.path.join(
corpus_path, "training_feat_part_" + str(rank) + ".lmdb")
else:
lmdb_file = os.path.join(corpus_path,
"gqa_resnext152_faster_rcnn_genome.lmdb")
# lmdb_file = os.path.join(corpus_path, "validation_feat_all.lmdb")
print("Loading from %s" % lmdb_file)
ds = td.LMDBSerializer.load(lmdb_file, shuffle=False)
self.num_dataset = len(ds)
ds = td.LocallyShuffleData(ds, cache)
caption_path = os.path.join(corpus_path, "caption_train.json")
# caption_path = os.path.join(corpus_path, "caption_val.json")
preprocess_function = BertPreprocessBatch(
caption_path,
tokenizer,
bert_model,
seq_len,
36,
self.num_dataset,
encoding="utf-8",
visual_target=visual_target,
objective=objective,
)
ds = td.PrefetchData(ds, 5000, 1)
ds = td.MapData(ds, preprocess_function)
# self.ds = td.PrefetchData(ds, 1)
ds = td.PrefetchDataZMQ(ds, num_workers)
self.ds = td.BatchData(ds, batch_size)
# self.ds = ds
self.ds.reset_state()
self.batch_size = batch_size
self.num_workers = num_workers
imgaug.Albumentations(AB.GaussianBlur(blur_limit=7, p=0.5)),
# imgaug.Albumentations(AB.GaussNoise(var_limit=(10.0, 50.0), p=0.5)),
]
ag_train = [
# imgaug.Resize(int(SHAPE * 1.12)),
# imgaug.RandomCrop(SHAPE),
# imgaug.Resize(int(SHAPE)),
imgaug.Flip(horiz=True),
imgaug.Flip(vert=True),
imgaug.Albumentations(AB.RandomRotate90(p=1))
]
# ds_train = df.AugmentImageComponent(ds_train, ag_image, 0) # Apply for image only
ds_train = df.AugmentImageComponents(ds_train, ag_train, (0, 1))
ds_train = df.MapData(
ds_train, lambda dp: [
np.expand_dims(dp[0], axis=0),
np.expand_dims(dp[1], axis=0),
])
ds_train = df.MultiProcessRunner(ds_train, num_proc=8, num_prefetch=4)
ds_train = df.BatchData(ds_train, batch_size=BATCH)
ds_train = df.PrintData(ds_train)
ds_train = df.MapData(
ds_train, lambda dp: [
torch.tensor(dp[0]),
torch.tensor(dp[1]),
])
# ds_valid
ds_valid = CustomDataFlow(size=500, datadir=args.data, istrain=False)
ag_valid = [
# configure logger
log_format = '[%(asctime)s %(levelname)s] %(message)s'
level = logging.DEBUG if args.debug else logging.INFO
if not args.log:
logging.basicConfig(level=level, format=log_format, stream=sys.stderr)
else:
logging.basicConfig(level=level, format=log_format, filename=args.log)
logging.info('args: %s', args)
if args.name == 'kinetics':
# get every frame of
ds = Kinetics(kinetics_dirpath, num_frames=1, skips=[0], shuffle=False)
# keep only first frame of each sub-video
# [sub_video_idx, frames[]] -> [first_frame, sub_video_idx]
ds = df.MapData(ds, lambda dp: [dp[1][0], dp[0]])
else:
ds = df.dataset.Cifar10('train', shuffle=False)
logging.info('Downsampling frames to 32x32 resolution')
ds = df.MapDataComponent(ds, lambda image: cv2.resize(image, (32, 32)))
logging.info('Converting RGB to Lab color space')
ds = df.MapDataComponent(ds, lambda image: cv2.cvtColor(np.float32(image / 255.0), cv2.COLOR_RGB2Lab))
ds = df.MapDataComponent(ds, lambda image: image[:, :, 1:])
ds = df.MapDataComponent(ds, lambda image: image.reshape((-1, 2)))
ds = df.RepeatedData(ds, -1)
ds.reset_state()
generator = ds.get_data()
samples = []
def dataflow(centroids, num_refs=3, num_process=16, shuffle=False):
"""
Compute graph to retrieve 3 reference and 1 target frames from Kinetics.
Downsample grayscale frames to 256x256 and colorized frames to 32x32
feature maps in Lab colorspace. Cluster colors in colorized frames.
Returned tensors are of shape (num_refs + 1, 256, 256, 1)
and (num_refs + 1, 32, 32, 1) each. Instead of colorized output,
cluster centroid index is returned.
:return: (grayscale input, cluster indices for colorized output)
"""
config = Config.get_instance()
kinetics_dirpath = config['data_dir']['kinetics']
# get frame and 3 prior reference frames with certain number of skips
data = Kinetics(kinetics_dirpath,
num_frames=num_refs + 1,
skips=[0, 4, 4, 8][:num_refs + 1],
shuffle=shuffle)
# downsample frames to 256x256
data = df.MapDataComponent(data,
ImageProcessor.resize(small_axis=256),
index=1)
data = df.MapDataComponent(data,
ImageProcessor.crop(shape=(256, 256)),
index=1)
# data = df.MapDataComponent(
# data, lambda images: [cv2.resize(image, (256, 256)) for image in images], index=1)
# split frames into 3 references and 1 target frame
# create deep copies of each at odd indices
data = df.MapData(
data, lambda dp: [
dp[1][:num_refs],
copy.deepcopy(dp[1][:num_refs]), dp[1][num_refs:],
copy.deepcopy(dp[1][num_refs:])
])
# decolorize first set of reference and target frames as (256, 256, 1)
for idx in [0, 2]:
data = df.MapDataComponent(
data,
lambda images: [
np.int32(cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)).reshape(
256, 256, 1) for image in images
],
index=idx)
for idx in [1, 3]:
# downsample to 32x32 feature map
data = df.MapDataComponent(
data,
lambda images: [cv2.resize(image, (32, 32)) for image in images],
index=idx)
# discard grayscale L space, keep only 'ab' from Lab color space
# scale from 0-255 to 0-1 for clustering in next step
data = df.MapDataComponent(
data,
lambda images: [
cv2.cvtColor(np.float32(image / 255.0), cv2.COLOR_BGR2Lab)
[:, :, 1:] for image in images
],
index=idx)
# find nearest color cluster index for every pixel in ref and target
data = df.MapDataComponent(
data,
lambda images:
[get_cluster_labels(image, centroids) for image in images],
index=idx)
# combine ref and target frames into (num_refs + 1, dim, dim, 1) tensor
# for both grayscale and colorized feature maps respectively
# generates [input tensor, output tensor]
data = df.MapData(
data, lambda dp:
[np.stack(dp[0] + dp[2], axis=0),
np.stack(dp[1] + dp[3], axis=0)])
# important for tensorflow.data.dataset
# does not do what it is supposed to do
data = df.MapData(data, tuple)
# prefetch 256 datapoints
data = df.MultiProcessPrefetchData(data,
nr_prefetch=256,
nr_proc=num_process)
data = df.PrefetchDataZMQ(data, nr_proc=1)
return data
