def dataflow(name='davis', scale=1, split='val'): if name == 'davis': ds = Davis('/data/zubin/videos/davis', name=split, num_frames=1, shuffle=False) elif name == 'kinetics': ds = Kinetics('/data/public/rw/datasets/videos/kinetics', num_frames=1, skips=[0], shuffle=False) else: raise Exception('not support dataset %s' % name) if name != 'davis': ds = df.MapData(ds, lambda dp: [dp[0], dp[1], dp[1]]) ds = df.MapData(ds, lambda dp: [ dp[0], # index dp[1], # original dp[2], # mask dp[3], # name ]) feature_size = int(256 * scale) size = (feature_size, feature_size) ds = df.MapDataComponent(ds, ImageProcess.resize(small_axis=feature_size), index=1) ds = df.MapDataComponent(ds, lambda images: cv2.resize(images[0], size), index=2) ds = df.MapData(ds, lambda dp: [ dp[0], # index dp[1][0], # original small axis 256 x scale cv2.cvtColor(cv2.resize(dp[1][0], size), cv2.COLOR_BGR2GRAY).reshape((size[0], size[1], 1)), # gray (256xscale)x(256xscale)x1 dp[2], # annotated mask 256xscale x 256xscale dp[3], # name ]) ds = df.MultiProcessPrefetchData(ds, nr_prefetch=32, nr_proc=1) return ds
def dataflow2(num_reference=3, shuffle=True): kinetics_path = "/media/engin/63c43c7a-cb63-4c43-b70c-f3cb4d68762a/datasets/kinetics/kinetics700" ds = KineticsClustered(kinetics_path, num_frames=num_reference + 1, skips=[0, 4, 4, 4][:num_reference + 1], shuffle=False) # ds = df.MapDataComponent(ds, lambda images: [cv2.resize(image, (256, 256)) for image in images], index=1) ds = df.MapData( ds, lambda dp: [ dp[1][:num_reference], dp[2][:num_reference], dp[1][ num_reference:], dp[2][num_reference:] ]) # # for images (ref, target) # for idx in [0, 2]: # ds = df.MapDataComponent(ds, lambda images: [cv2.cvtColor(image, cv2.COLOR_BGR2GRAY).reshape(256, 256, 1) for image in images], index=idx) # stack for tensor ds = df.MapData( ds, lambda dp: [np.stack(dp[0] + dp[2], axis=0), np.stack(dp[1] + dp[3], axis=0)]) ds = df.MapData(ds, tuple) # for tensorflow.data.dataset # ds = df.MultiProcessPrefetchData(ds, nr_prefetch=256, nr_proc=num_process) # ds = df.PrefetchtraDataZMQ(ds, nr_proc=1) # ds = df.RepeatedData(ds, total_num_epoch) # ds = df.RepeatedData(ds, -1) return ds
def dataflow(centroids, num_reference=3, num_process=16, shuffle=True): ds = Kinetics('/data/public/rw/datasets/videos/kinetics', num_frames=num_reference + 1, skips=[0, 4, 4, 8][:num_reference + 1], shuffle=shuffle) ds = df.MapDataComponent(ds, ImageProcess.resize(small_axis=256), index=1) ds = df.MapDataComponent(ds, ImageProcess.crop(shape=(256, 256)), index=1) #ds = df.MapDataComponent(ds, lambda images: [cv2.resize(image, (256, 256)) for image in images], index=1) ds = df.MapData( ds, lambda dp: [ dp[1][:num_reference], copy.deepcopy(dp[1][:num_reference]), dp[1][num_reference:], copy.deepcopy(dp[1][num_reference:]) ]) # for images (ref, target) for idx in [0, 2]: ds = df.MapDataComponent( ds, lambda images: [ cv2.cvtColor(image, cv2.COLOR_BGR2GRAY).reshape(256, 256, 1) for image in images ], index=idx) # for labels (ref, target) for idx in [1, 3]: ds = df.MapDataComponent( ds, lambda images: [cv2.resize(image, (32, 32)) for image in images], index=idx) ds = df.MapDataComponent(ds, lambda images: [ cv2.cvtColor(np.float32(image / 255.0), cv2.COLOR_BGR2Lab)[:, :, 1:] for image in images ], index=idx) ds = df.MapDataComponent( ds, lambda images: [ np.array([ np.argmin(np.linalg.norm(centroids - v, axis=1)) for v in image.reshape((-1, 2)) ]).reshape((32, 32, 1)) for image in images ], index=idx) # stack for tensor ds = df.MapData( ds, lambda dp: [np.stack(dp[0] + dp[2], axis=0), np.stack(dp[1] + dp[3], axis=0)]) ds = df.MapData(ds, tuple) # for tensorflow.data.dataset ds = df.MultiProcessPrefetchData(ds, nr_prefetch=256, nr_proc=num_process) ds = df.PrefetchDataZMQ(ds, nr_proc=1) return ds
def get_input_fn(name, centroids, batch_size=32, num_refs=3, num_process=16, shuffle=False): """Iterate over datapoints in dataflow in mini-batches.""" _ = name data = dataflow(centroids, num_refs=num_refs, num_process=num_process, shuffle=shuffle) data = df.MapData(data, tuple) data.reset_state() def input_fn(): with tf.name_scope('dataset'): dataset = tf.data.Dataset.from_generator( data.get_data, output_types=(tf.int32, tf.int32), output_shapes=(tf.TensorShape([num_refs + 1, 256, 256, 1]), tf.TensorShape([num_refs + 1, 32, 32, 1]))).batch(batch_size) return dataset return input_fn
def get_input_fn(name, batch_size=32): image_size = 32 is_training = name == 'train' ds = df.dataset.Cifar10(name, shuffle=is_training) ds = df.MapDataComponent( ds, lambda x: np.pad(x, [(4, 4), (4, 4), (0, 0)], mode='reflect'), index=0) augmentors = [ tp.imgaug.RandomCrop((32, 32)), tp.imgaug.Flip(horiz=True), #tp.imgaug.MapImage(lambda x: (x - pp_mean)/128.0), ] if is_training: ds = df.RepeatedData(ds, -1) ds = tp.AugmentImageComponent(ds, augmentors) else: ds = tp.AugmentImageComponent(ds, [tp.imgaug.CenterCrop((32, 32))]) ds = tp.AugmentImageComponent(ds, [tp.imgaug.Resize((image_size, image_size))]) ds = df.MapData(ds, tuple) # for tensorflow.data.dataset ds.reset_state() def input_fn(): with tf.name_scope('dataset'): dataset = tf.data.Dataset.from_generator( ds.get_data, output_types=(tf.float32, tf.int64), output_shapes=(tf.TensorShape([image_size, image_size, 3]), tf.TensorShape([]))).batch(batch_size) return dataset return input_fn
def __init__(self, bert_model_name, captions, images, dataset_type, image_features, obj_list, seq_len, encoding="utf-8"): self.tokenizer = BertTokenizer.from_pretrained(bert_model_name, do_lower_case=True) self.dataset_type = dataset_type self.imageid2filepath = {} entries = self._load_entries(images, captions, dataset_type) preprocess_function = BertPreprocessBatch( tokenizer=self.tokenizer, obj_list=obj_list, seq_len=seq_len, region_len=36, image_features=image_features, imageid2filepath=self.imageid2filepath, encoding=encoding, predict_feature=False) ds = td.MapData(entries, preprocess_function) self.formatted_entries = list(ds.get_data())
def __init__(self, corpus_path, tokenizer, seq_len, encoding="utf-8", predict_feature=False, batch_size=512, shuffle=False, num_workers=25, cache=10000, drop_last=False, cuda=False, distributed=False, visualization=False, span_mask=False, cond_mask=False, region_len=36): if dist.is_available() and distributed: rank = dist.get_rank() lmdb_file = os.path.join( corpus_path, "training_feat_part_" + str(rank) + ".lmdb") else: lmdb_file = os.path.join(corpus_path, "training_feat_all.lmdb") caption_path = os.path.join(corpus_path, "caption_train.json") print("Loading from %s" % lmdb_file) os.listdir(corpus_path) ds = td.LMDBSerializer.load(lmdb_file, shuffle=False) self.num_dataset = len(ds) self.cond_mask = cond_mask preprocess_function = BertPreprocessBatch( caption_path, tokenizer, seq_len, region_len, self.num_dataset, encoding="utf-8", predict_feature=predict_feature, span_mask=span_mask, cond_mask=cond_mask) # ds = td.LocallyShuffleData(ds, cache) ds = td.PrefetchData(ds, 5000, 1) ds = td.MapData(ds, preprocess_function) # self.ds = td.PrefetchData(ds, 1) ds = td.PrefetchDataZMQ(ds, num_workers) self.ds = td.BatchData(ds, batch_size) # self.ds = ds self.ds.reset_state() self.batch_size = batch_size self.num_workers = num_workers
def __init__( self, corpus_path, tokenizer, seq_len, encoding='utf-8', predict_feature=False, hard_negative=False, batch_size=512, shuffle=False, num_workers=25, cache=50000, drop_last=False, cuda=False, distributed=False, visualization=False, ): if dist.is_available() and distributed: # num_replicas = dist.get_world_size() # assert num_replicas == 8 rank = dist.get_rank() lmdb_file = '/mnt3/xuesheng/features_lmdb/CC/training_feat_part_' + str( rank) + '.lmdb' # if not os.path.exists(lmdb_file): # lmdb_file = "/srv/share/datasets/conceptual_caption/training_feat_part_" + str(rank) + ".lmdb" else: # lmdb_file = "/coc/dataset/conceptual_caption/training_feat_all.lmdb" # if not os.path.exists(lmdb_file): lmdb_file = '/mnt3/xuesheng/features_lmdb/CC/training_feat_part_0.lmdb' caption_path = '/mnt3/xuesheng/features_lmdb/CC/caption_train.json' print('Loading from %s' % lmdb_file) ds = td.LMDBSerializer.load(lmdb_file, shuffle=True) self.num_dataset = len(ds) preprocess_function = BertPreprocessBatch( caption_path, tokenizer, seq_len, 36, self.num_dataset, encoding='utf-8', predict_feature=predict_feature, ) # ds = td.LocallyShuffleData(ds, cache) # ds = td.PrefetchData(ds, 5000, 1) ds = td.MapData(ds, preprocess_function) # self.ds = td.PrefetchData(ds, 1) ds = td.PrefetchDataZMQ(ds, num_workers) self.ds = td.BatchData(ds, batch_size) # self.ds = ds self.ds.reset_state() self.batch_size = batch_size self.num_workers = num_workers
def dataflow4(num_reference=3, order=1, shuffle=True): kinetics_path = "/media/engin/63c43c7a-cb63-4c43-b70c-f3cb4d68762a/datasets/kinetics/kinetics700" ds = KineticsClustered(order, kinetics_path, num_frames=num_reference + 1, skips=[0, 4, 4, 4][:num_reference + 1], shuffle=False) # # stack for tensor ds = df.MapData( ds, lambda dp: [np.stack(dp[1], axis=0), np.stack(dp[2], axis=0)]) ds = df.MapData(ds, tuple) # for tensorflow.data.dataset ds = df.RepeatedData(ds, -1) return ds
def dataflow3(num_reference=3, num_sets=1, shuffle=True): kinetics_path = "/media/engin/63c43c7a-cb63-4c43-b70c-f3cb4d68762a/datasets/kinetics/kinetics700" ds_list = [] for i in range(num_sets): ds1 = KineticsClustered(i, kinetics_path, num_frames=num_reference + 1, skips=[0, 4, 4, 4][:num_reference + 1], shuffle=False) ds1 = df.RepeatedData(ds1, -1) ds_list.append(ds1) # ds2 = KineticsClustered(1, kinetics_path, num_frames=num_reference + 1, # skips=[0, 4, 4, 4][:num_reference + 1], shuffle=False) # ds2 = df.RepeatedData(ds2, -1) ds = df.JoinData(ds_list) # ds = df.MapData(ds, lambda dp: [ [dp[0], dp[1], dp[2]] ]) ds = df.MapData( ds, lambda dp: [[dp[i], dp[i + 1], dp[i + 2]] for i in range(0, num_sets * 3, 3)]) # for idx in [0, 1]: # ds = df.MapDataComponent(ds, lambda dp: [dp[1][:num_reference], dp[2][:num_reference], dp[1][num_reference:], dp[2][num_reference:]], index=idx) # # stack for tensor for idx in range(num_sets): ds = df.MapDataComponent( ds, lambda dp: [np.stack(dp[1], axis=0), np.stack(dp[2], axis=0)], index=idx) ds = df.MapData(ds, tuple) # for tensorflow.data.dataset # ds = df.BatchData(ds, 2, use_list=True) #Prepare epochs # ds = df.RepeatedData(ds, total_num_epoch) # ds = df.RepeatedData(ds, -1) return ds
def dataflow(name='davis', scale=1): if name == 'davis': ds = Davis('/data/public/rw/datasets/videos/davis/trainval', num_frames=1, shuffle=False) elif name == 'kinetics': ds = Kinetics('/data/public/rw/datasets/videos/kinetics', num_frames=1, skips=[0], shuffle=False) else: raise Exception('not support dataset %s' % name) if name != 'davis': ds = df.MapData(ds, lambda dp: [dp[0], dp[1], dp[1]]) ds = df.MapData( ds, lambda dp: [ dp[0], # index dp[1], # original dp[2], # mask ]) size = (256 * scale, 256 * scale) ds = df.MapDataComponent(ds, ImageProcess.resize(small_axis=256 * scale), index=1) ds = df.MapDataComponent(ds, lambda images: cv2.resize(images[0], size), index=2) ds = df.MapData( ds, lambda dp: [ dp[0], # index dp[1][0], # original cv2.cvtColor(cv2.resize(dp[1][0], size), cv2.COLOR_BGR2GRAY). reshape((size[0], size[1], 1)), # gray dp[2], # mask ]) ds = df.MultiProcessPrefetchData(ds, nr_prefetch=32, nr_proc=1) return ds
def parallel(self, num_threads, buffer_size=200, strict=False): self.is_parallel = True ds = self ds = df.MultiThreadMapData(ds, nr_thread=num_threads, map_func=NetworkImages.map_func_download, buffer_size=buffer_size, strict=strict) # ds = df.PrefetchDataZMQ(ds, nr_proc=1) # to reduce GIL contention. ds = df.MapData(ds, func=NetworkImages.map_func_decode) return ds
def create_dataflow(data_dir: Path, kind: str, batch_size: int, shuffle: bool = True) -> td.DataFlow: path = data_dir / "{}.mdb".format(kind) ds = td.LMDBData(str(path), shuffle=shuffle) ds = td.MapData(ds, _decode_data) ds = td.BatchData(ds, batch_size, remainder=False) ds = td.MapDataComponent(ds, _squeeze_last, index=1) return ds
def dataflow(name='davis', scale=1): """Compute graph to retrieve index, grayscale index, annotation.""" cfg = Config.get_instance() # get test index one at a time if name == 'davis': data_dirpath = cfg['data_dir']['davis'] data = Davis(data_dirpath, num_frames=1, shuffle=False) elif name == 'kinetics': data_dirpath = cfg['data_dir']['kinetics'] data = Kinetics(data_dirpath, num_frames=1, skips=[0], shuffle=False) else: raise Exception('Dataset [%s] not supported.' % name) # repeat Kinetics index since Davis has image and annotated frames if name != 'davis': data = df.MapData(data, lambda dp: [dp[0], dp[1], dp[1]]) data = df.MapData(data, lambda dp: [dp[0], dp[1], dp[2]]) length = 256 * scale size = (length, length) # resize frames to 256x256 data = df.MapDataComponent(data, ImageProcessor.resize(small_axis=length), index=1) data = df.MapDataComponent(data, lambda images: cv2.resize(images[0], size), index=2) # get index, original index, gray scale index, annotation mask data = df.MapData( data, lambda dp: [ dp[0], dp[1][0], cv2.cvtColor(cv2.resize(dp[1][0], size), cv2.COLOR_BGR2GRAY). reshape((length, length, 1)), dp[2], ]) data = df.MultiProcessPrefetchData(data, nr_prefetch=32, nr_proc=1) return data
def __init__(self, datafile, batch_size, num_workers=1, nviews=12, reset=True, augment=False, filter_classes=None, filter_views=None, polarmode='cartesian', shuffle=True, filter_ids=None, label_to0idx=False, rgb=False, force_res=0, autocrop=False, keep_aspect_ratio=False): self.filter_classes = filter_classes self.filter_views = filter_views self.filter_ids = filter_ids self.polarmode = polarmode self.label_to0idx = label_to0idx self.rgb = rgb self.force_res = force_res self.autocrop = autocrop self.keep_aspect_ratio = keep_aspect_ratio if not isinstance(datafile, list): datafile = [datafile] ds = [] for d in datafile: ds.append(df.LMDBSerializer.load(d, shuffle=shuffle)) if shuffle: ds[-1] = df.LocallyShuffleData(ds[-1], 100) ds[-1] = df.PrefetchData(ds[-1], 20, 1) ds[-1] = df.MapData(ds[-1], self.load) if augment: ds[-1] = df.MapDataComponent(ds[-1], LMDBMultiView._augment, 0) if (not filter_classes and not filter_ids and num_workers > 1): # warning: skipping this is slower when filtering datasets # but epoch counting will be wrong otherwise ds[-1] = df.PrefetchDataZMQ(ds[-1], num_workers) ds[-1] = df.BatchData(ds[-1], batch_size) if reset: ds[-1].reset_state() self.ds = ds
def __init__( self, annotations_path, features_path, tokenizer, bert_model, seq_len, batch_size=512, num_workers=25, cache=10000, local_rank=-1, objective=0, num_locs=5, add_global_imgfeat=None, ): if dist.is_available() and local_rank != -1: rank = dist.get_rank() lmdb_file = os.path.join( features_path, "training_feat_part_" + str(rank) + ".lmdb") else: lmdb_file = os.path.join(features_path, "training_feat_all.lmdb") print("Loading from %s" % lmdb_file) ds = td.LMDBSerializer.load(lmdb_file, shuffle=False) self.num_dataset = len(ds) ds = td.LocallyShuffleData(ds, cache) caption_path = os.path.join(annotations_path, "caption_train.json") preprocess_function = BertPreprocessBatch( caption_path, tokenizer, bert_model, seq_len, 36, self.num_dataset, objective=objective, num_locs=num_locs, ) ds = td.PrefetchData(ds, 5000, 1) ds = td.MapData(ds, preprocess_function) ds = td.PrefetchDataZMQ(ds, num_workers) self.ds = td.BatchData(ds, batch_size) self.ds.reset_state() self.batch_size = batch_size self.num_workers = num_workers self.add_global_imgfeat = add_global_imgfeat self.num_locs = num_locs
def __init__(self, corpus_path, tokenizer, seq_len, encoding="utf-8", predict_feature=False, batch_size=512, shuffle=False, num_workers=25, cache=50000, drop_last=False, cuda=False, distributed=False, visualization=False, span_mask=False, cond_mask=False, region_len=36): lmdb_file = os.path.join(corpus_path, "validation_all.lmdb") caption_path = os.path.join(corpus_path, "caption_val.json") print("Loading from %s" % lmdb_file) ds = td.LMDBSerializer.load(lmdb_file, shuffle=False) self.num_dataset = len(ds) self.cond_mask = cond_mask preprocess_function = BertPreprocessBatch( caption_path, tokenizer, seq_len, region_len, self.num_dataset, encoding="utf-8", predict_feature=predict_feature, visualization=visualization, span_mask=span_mask, cond_mask=cond_mask, ) ds = td.MapData(ds, preprocess_function) self.ds = td.BatchData(ds, batch_size) self.ds.reset_state() self.batch_size = batch_size self.num_workers = num_workers
def init_dataflow(ctfstar,batch_size): ''' This function creates dataflow that reads and preprocesses data in parallel ''' augm = df.imgaug.AugmentorList([df.imgaug.MeanVarianceNormalize()]) # create partitioned generators, one for each element in a batch dss0,shape = MicrosGenerator.create_partition(ctfstar,batch_size) # preprocess input dss1 = [df.MapData(ds0, lambda dp: [augm.augment(preprocess_micro(dp[0], dp[1], psize, bn)), np.array(dp[0])]) for ds0 in dss0] # prefetch each generator in a separate process with buffer of 4 images per process # dss1 = [df.PrefetchDataZMQ(ds1, nr_proc=1, hwm=2) for ds1 in dss1] dss1 = [df.PrefetchData(ds1, nr_prefetch=4, nr_proc=1) for ds1 in dss1] # join all dataflows ds1 = df.RandomMixData(dss1) # ds1 = df.JoinData(dss1) ds = df.BatchData(ds1, batch_size) ds.reset_state() return ds,shape
def __init__( self, corpus_path, tokenizer, seq_len, encoding='utf-8', predict_feature=False, batch_size=512, shuffle=False, num_workers=25, cache=50000, drop_last=False, cuda=False, distributed=False, visualization=False, ): lmdb_file = '/mnt3/xuesheng/features_lmdb/CC_val/validation_all.lmdb' if not os.path.exists(lmdb_file): lmdb_file = '/mnt3/xuesheng/features_lmdb/CC_val/validation_all.lmdb' caption_path = '/mnt3/xuesheng/features_lmdb/CC_val/caption_val.json' print('Loading from %s' % lmdb_file) ds = td.LMDBSerializer.load(lmdb_file, shuffle=False) self.num_dataset = len(ds) preprocess_function = BertPreprocessBatch( caption_path, tokenizer, seq_len, 36, self.num_dataset, encoding='utf-8', predict_feature=predict_feature, visualization=visualization, ) ds = td.MapData(ds, preprocess_function) self.ds = td.BatchData(ds, batch_size) self.ds.reset_state() self.batch_size = batch_size self.num_workers = num_workers
def __init__( self, corpus_path, tokenizer, seq_len, encoding="utf-8", predict_feature=False, batch_size=512, shuffle=False, num_workers=25, cache=50000, drop_last=False, cuda=False, distributed=False, visualization=False, ): lmdb_file = "/coc/dataset/conceptual_caption/validation_feat_all.lmdb" if not os.path.exists(lmdb_file): lmdb_file = "/coc/pskynet2/jlu347/multi-modal-bert/data/conceptual_caption/validation_feat_all.lmdb" caption_path = "/coc/pskynet2/jlu347/multi-modal-bert/data/conceptual_caption/caption_val.json" print("Loading from %s" % lmdb_file) ds = td.LMDBSerializer.load(lmdb_file, shuffle=False) self.num_dataset = len(ds) preprocess_function = BertPreprocessBatch( caption_path, tokenizer, seq_len, 36, self.num_dataset, encoding="utf-8", predict_feature=predict_feature, visualization=visualization, ) ds = td.MapData(ds, preprocess_function) self.ds = td.BatchData(ds, batch_size) self.ds.reset_state() self.batch_size = batch_size self.num_workers = num_workers
def __init__( self, corpus_path, tokenizer, bert_model, seq_len, encoding="utf-8", visual_target=0, batch_size=512, shuffle=False, num_workers=25, cache=5000, drop_last=False, cuda=False, objective=0, visualization=False, ): lmdb_file = os.path.join(corpus_path, "validation_feat_all.lmdb") caption_path = os.path.join(corpus_path, "caption_val.json") print("Loading from %s" % lmdb_file) ds = td.LMDBSerializer.load(lmdb_file, shuffle=False) self.num_dataset = len(ds) preprocess_function = BertPreprocessBatch( caption_path, tokenizer, bert_model, seq_len, 36, self.num_dataset, encoding="utf-8", visual_target=visual_target, visualization=visualization, objective=objective, ) ds = td.MapData(ds, preprocess_function) self.ds = td.BatchData(ds, batch_size) self.ds.reset_state() self.batch_size = batch_size self.num_workers = num_workers
def __init__( self, annotations_path, features_path, tokenizer, bert_model, seq_len, batch_size=512, num_workers=25, cache=5000, objective=0, num_locs=5, add_global_imgfeat=True, visualization=False, ): lmdb_file = os.path.join(features_path, "validation_feat_all.lmdb") caption_path = os.path.join(annotations_path, "caption_valid.json") print("Loading from %s" % lmdb_file) ds = td.LMDBSerializer.load(lmdb_file, shuffle=False) self.num_dataset = len(ds) preprocess_function = BertPreprocessBatch( caption_path, tokenizer, bert_model, seq_len, 36, self.num_dataset, visualization=visualization, objective=objective, num_locs=num_locs, ) ds = td.MapData(ds, preprocess_function) self.ds = td.BatchData(ds, batch_size) self.ds.reset_state() self.batch_size = batch_size self.num_workers = num_workers self.add_global_imgfeat = add_global_imgfeat self.num_locs = num_locs
def lmdb_dataflow(lmdb_path, batch_size, input_size, output_size, is_training, test_speed=False, filter_rate=0): df = dataflow.LMDBSerializer.load(lmdb_path, shuffle=False) df = dataflow.MapData(df, lambda dp: [item for item in dp] + [random.random()]) size = df.size() print(size) if is_training: df = dataflow.LocallyShuffleData(df, buffer_size=2000) df = dataflow.PrefetchData(df, nr_prefetch=500, nr_proc=1) df = BatchData(df, batch_size, input_size, output_size) if is_training: df = dataflow.PrefetchDataZMQ(df, nr_proc=8) df = dataflow.RepeatedData(df, -1) if test_speed: dataflow.TestDataSpeed(df, size=1000).start() df.reset_state() return df, size
df.imgaug.CenterPaste((36, 36)), df.imgaug.RandomCrop((32, 32)), df.imgaug.RandomOrderAug([ df.imgaug.RandomApplyAug( df.imgaug.BrightnessScale((0.8, 1.2), clip=False), 0.5), df.imgaug.RandomApplyAug( df.imgaug.Contrast((0.8, 1.2), clip=False), 0.5), # df.imgaug.RandomApplyAug(df.imgaug.Saturation(0.4, rgb=False), 0.5), ]), ] augmentors_default = [ df.imgaug.Resize((32, 32)), df.imgaug.MapImage(lambda x: x.reshape(32, 32, 1)) ] # keep original image at index 1 ds = df.MapData( ds, lambda datapoint: [datapoint[0], datapoint[0]] + datapoint[1:]) ds = df.AugmentImageComponent(ds, augmentors_variation + augmentors_default) ds = df.AugmentImageComponent(ds, augmentors_default, index=1) ds = df.PrefetchData(ds, nr_prefetch=12, nr_proc=4) ds = df.PrintData(ds) ds = df.BatchData(ds, batch_size=32, remainder=False, use_list=True) ds = df.PrintData(ds) for minibatch in ds.get_data(): images, originals, labels = minibatch image, original, label = images[0], originals[0], labels[0] name = '{:02d}'.format(label) cv2.namedWindow(name) cv2.moveWindow(name, 0, 25 + 128 * label)
def __init__( self, corpus_path, tokenizer, seq_len, encoding='utf-8', predict_feature=False, batch_size=512, shuffle=False, num_workers=10, cache=50000, drop_last=False, cuda=False, ): lmdb_file = '/coc/dataset/conceptual_caption/validation_feat_all.lmdb' if not os.path.exists(lmdb_file): lmdb_file = '/coc/pskynet2/jlu347/multi-modal-bert/data/conceptual_caption/validation_feat_all.lmdb' caption_path = '/coc/pskynet2/jlu347/multi-modal-bert/data/conceptual_caption/caption_val.json' print('Loading from %s' % lmdb_file) ds = td.LMDBSerializer.load(lmdb_file, shuffle=False) self.num_dataset = len(ds) preprocess_function = BertPreprocessRetrieval( caption_path, tokenizer, seq_len, 36, 1000, encoding='utf-8', predict_feature=predict_feature, ) ds = td.MapData(ds, preprocess_function) self.ds = td.BatchData(ds, 1) self.ds.reset_state() self.batch_size = 1 self.num_workers = num_workers self._entry = [] self.features_all = np.zeros((1000, 37, 2048), dtype=np.float32) self.spatials_all = np.zeros((1000, 37, 5), dtype=np.float32) self.image_mask_all = np.zeros((1000, 37), dtype=np.float32) self.image_ids = [] # load first 1000 file here. for i, batch in enumerate(self.ds.get_data()): if i >= 1000: break input_ids, input_mask, segment_ids, is_next, image_feat, image_loc, image_mask, image_id, caption = batch batch_size = input_ids.shape[0] g_image_feat = np.sum(image_feat, axis=1) / np.sum( image_mask, axis=1, keepdims=True) image_feat = np.concatenate( [np.expand_dims(g_image_feat, axis=1), image_feat], axis=1) image_feat = np.array(image_feat, dtype=np.float32) g_image_loc = np.repeat(np.array([[0, 0, 1, 1, 1]], dtype=np.float32), batch_size, axis=0) image_loc = np.concatenate( [np.expand_dims(g_image_loc, axis=1), image_loc], axis=1) image_loc = np.array(image_loc, dtype=np.float32) g_image_mask = np.repeat(np.array([[1]]), batch_size, axis=0) image_mask = np.concatenate([g_image_mask, image_mask], axis=1) batch = (input_ids, input_mask, segment_ids, image_id, caption) self._entry.append(batch) self.features_all[i] = image_feat self.image_mask_all[i] = np.array(image_mask) self.spatials_all[i] = image_loc self.image_ids.append(image_id) sys.stdout.write('%d/%d\r' % (i, 1000)) sys.stdout.flush()
def __init__( self, corpus_path, tokenizer, bert_model, seq_len, encoding="utf-8", visual_target=0, hard_negative=False, batch_size=512, shuffle=False, num_workers=25, cache=10000, drop_last=False, cuda=False, local_rank=-1, objective=0, visualization=False, ): TRAIN_DATASET_SIZE = 3119449 if dist.is_available() and local_rank != -1: num_replicas = dist.get_world_size() rank = dist.get_rank() lmdb_file = os.path.join( corpus_path, "training_feat_part_" + str(rank) + ".lmdb") else: lmdb_file = os.path.join(corpus_path, "gqa_resnext152_faster_rcnn_genome.lmdb") # lmdb_file = os.path.join(corpus_path, "validation_feat_all.lmdb") print("Loading from %s" % lmdb_file) ds = td.LMDBSerializer.load(lmdb_file, shuffle=False) self.num_dataset = len(ds) ds = td.LocallyShuffleData(ds, cache) caption_path = os.path.join(corpus_path, "caption_train.json") # caption_path = os.path.join(corpus_path, "caption_val.json") preprocess_function = BertPreprocessBatch( caption_path, tokenizer, bert_model, seq_len, 36, self.num_dataset, encoding="utf-8", visual_target=visual_target, objective=objective, ) ds = td.PrefetchData(ds, 5000, 1) ds = td.MapData(ds, preprocess_function) # self.ds = td.PrefetchData(ds, 1) ds = td.PrefetchDataZMQ(ds, num_workers) self.ds = td.BatchData(ds, batch_size) # self.ds = ds self.ds.reset_state() self.batch_size = batch_size self.num_workers = num_workers
imgaug.Albumentations(AB.GaussianBlur(blur_limit=7, p=0.5)), # imgaug.Albumentations(AB.GaussNoise(var_limit=(10.0, 50.0), p=0.5)), ] ag_train = [ # imgaug.Resize(int(SHAPE * 1.12)), # imgaug.RandomCrop(SHAPE), # imgaug.Resize(int(SHAPE)), imgaug.Flip(horiz=True), imgaug.Flip(vert=True), imgaug.Albumentations(AB.RandomRotate90(p=1)) ] # ds_train = df.AugmentImageComponent(ds_train, ag_image, 0) # Apply for image only ds_train = df.AugmentImageComponents(ds_train, ag_train, (0, 1)) ds_train = df.MapData( ds_train, lambda dp: [ np.expand_dims(dp[0], axis=0), np.expand_dims(dp[1], axis=0), ]) ds_train = df.MultiProcessRunner(ds_train, num_proc=8, num_prefetch=4) ds_train = df.BatchData(ds_train, batch_size=BATCH) ds_train = df.PrintData(ds_train) ds_train = df.MapData( ds_train, lambda dp: [ torch.tensor(dp[0]), torch.tensor(dp[1]), ]) # ds_valid ds_valid = CustomDataFlow(size=500, datadir=args.data, istrain=False) ag_valid = [
# configure logger log_format = '[%(asctime)s %(levelname)s] %(message)s' level = logging.DEBUG if args.debug else logging.INFO if not args.log: logging.basicConfig(level=level, format=log_format, stream=sys.stderr) else: logging.basicConfig(level=level, format=log_format, filename=args.log) logging.info('args: %s', args) if args.name == 'kinetics': # get every frame of ds = Kinetics(kinetics_dirpath, num_frames=1, skips=[0], shuffle=False) # keep only first frame of each sub-video # [sub_video_idx, frames[]] -> [first_frame, sub_video_idx] ds = df.MapData(ds, lambda dp: [dp[1][0], dp[0]]) else: ds = df.dataset.Cifar10('train', shuffle=False) logging.info('Downsampling frames to 32x32 resolution') ds = df.MapDataComponent(ds, lambda image: cv2.resize(image, (32, 32))) logging.info('Converting RGB to Lab color space') ds = df.MapDataComponent(ds, lambda image: cv2.cvtColor(np.float32(image / 255.0), cv2.COLOR_RGB2Lab)) ds = df.MapDataComponent(ds, lambda image: image[:, :, 1:]) ds = df.MapDataComponent(ds, lambda image: image.reshape((-1, 2))) ds = df.RepeatedData(ds, -1) ds.reset_state() generator = ds.get_data() samples = []
def dataflow(centroids, num_refs=3, num_process=16, shuffle=False): """ Compute graph to retrieve 3 reference and 1 target frames from Kinetics. Downsample grayscale frames to 256x256 and colorized frames to 32x32 feature maps in Lab colorspace. Cluster colors in colorized frames. Returned tensors are of shape (num_refs + 1, 256, 256, 1) and (num_refs + 1, 32, 32, 1) each. Instead of colorized output, cluster centroid index is returned. :return: (grayscale input, cluster indices for colorized output) """ config = Config.get_instance() kinetics_dirpath = config['data_dir']['kinetics'] # get frame and 3 prior reference frames with certain number of skips data = Kinetics(kinetics_dirpath, num_frames=num_refs + 1, skips=[0, 4, 4, 8][:num_refs + 1], shuffle=shuffle) # downsample frames to 256x256 data = df.MapDataComponent(data, ImageProcessor.resize(small_axis=256), index=1) data = df.MapDataComponent(data, ImageProcessor.crop(shape=(256, 256)), index=1) # data = df.MapDataComponent( # data, lambda images: [cv2.resize(image, (256, 256)) for image in images], index=1) # split frames into 3 references and 1 target frame # create deep copies of each at odd indices data = df.MapData( data, lambda dp: [ dp[1][:num_refs], copy.deepcopy(dp[1][:num_refs]), dp[1][num_refs:], copy.deepcopy(dp[1][num_refs:]) ]) # decolorize first set of reference and target frames as (256, 256, 1) for idx in [0, 2]: data = df.MapDataComponent( data, lambda images: [ np.int32(cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)).reshape( 256, 256, 1) for image in images ], index=idx) for idx in [1, 3]: # downsample to 32x32 feature map data = df.MapDataComponent( data, lambda images: [cv2.resize(image, (32, 32)) for image in images], index=idx) # discard grayscale L space, keep only 'ab' from Lab color space # scale from 0-255 to 0-1 for clustering in next step data = df.MapDataComponent( data, lambda images: [ cv2.cvtColor(np.float32(image / 255.0), cv2.COLOR_BGR2Lab) [:, :, 1:] for image in images ], index=idx) # find nearest color cluster index for every pixel in ref and target data = df.MapDataComponent( data, lambda images: [get_cluster_labels(image, centroids) for image in images], index=idx) # combine ref and target frames into (num_refs + 1, dim, dim, 1) tensor # for both grayscale and colorized feature maps respectively # generates [input tensor, output tensor] data = df.MapData( data, lambda dp: [np.stack(dp[0] + dp[2], axis=0), np.stack(dp[1] + dp[3], axis=0)]) # important for tensorflow.data.dataset # does not do what it is supposed to do data = df.MapData(data, tuple) # prefetch 256 datapoints data = df.MultiProcessPrefetchData(data, nr_prefetch=256, nr_proc=num_process) data = df.PrefetchDataZMQ(data, nr_proc=1) return data