def getdata(path, isTrain): ds = LMDBSerializer.load(path, shuffle=isTrain) # Graph Benchmark # ds=FakeData([[10,10],[10,10],[10,10],[10,10],[10],[10],[10,10],[1],[1],[1]], 1000, random=False,dtype=['int32', 'int32', 'int32', 'int32', 'int32', 'int32',\ # 'int32', 'int32', 'int32', 'int32'], domain=[(0, 100), (0, 120),(0,120),(0,1),(0,100),(0,100),(0,100),(0,52),(0,115),(0,115)]) ds = getbatch(ds, 32, isTrain) if isTrain: ds = MultiProcessRunnerZMQ(ds, 4) return ds
def compute_mean_std(db, fname): ds = LMDBSerializer.load(db, shuffle=False) ds.reset_state() o = OnlineMoments() for dp in get_tqdm(ds): feat = dp[0] # len x dim for f in feat: o.feed(f) logger.info("Writing to {} ...".format(fname)) with open(fname, 'wb') as f: f.write(serialize.dumps([o.mean, o.std]))
def compute_mean_std(db, fname): ds = LMDBSerializer.load(db, shuffle=False) ds.reset_state() o = OnlineMoments() for dp in get_tqdm(ds): feat = dp[0] # len x dim for f in feat: o.feed(f) logger.info("Writing to {} ...".format(fname)) with open(fname, 'wb') as f: f.write(serialize.dumps([o.mean, o.std]))
def __init__(self, file_location, batch_size, train=True, shuffle=True, full=False, batch_from_disk=150): self.batch_size = batch_size self.train = train if train: self.ds = MyLMDBSerializer.load(file_location, shuffle=shuffle, batch_from_disk=batch_from_disk) self.ds = MyLocallyShuffleData(self.ds, buffer_size=10000, shuffle_interval=500) self.ds = MultiProcessRunnerZMQ(self.ds, num_proc=1, hwm=10000) self.len_ = 1281167 else: self.ds = LMDBSerializer.load(file_location, shuffle=False) self.ds = MultiProcessRunnerZMQ(self.ds, num_proc=1, hwm=10000) self.len_ = 50000 self.ds.reset_state() self.batches_in_epoch = int(math.ceil(self.len_ / self.batch_size))
def __init__(self, mode, batch_size=256, shuffle=False, num_workers=25, cache=50000, device='cuda'): # enumerate standard imagenet augmentors imagenet_augmentors = fbresnet_augmentor(mode == 'train') # load the lmdb if we can find it base_dir = '/userhome/cs/u3003679/' lmdb_loc = os.path.join(base_dir, 'ILSVRC-{}.lmdb'.format(mode)) #lmdb_loc = os.path.join(os.environ['IMAGENET'], 'ILSVRC-%s.lmdb'%mode) ds = LMDBSerializer.load(lmdb_loc, shuffle=shuffle) ds = LocallyShuffleData(ds, cache) # ds = td.LMDBDataPoint(ds) def f(dp): x, label = dp x = cv2.imdecode(x, cv2.IMREAD_COLOR) for aug in imagenet_augmentors: x = aug.augment(x) return x, label ds = MultiProcessMapDataZMQ(ds, num_proc=8, map_func=f) # ds = MapDataComponent(ds, lambda x: cv2.imdecode(x, cv2.IMREAD_COLOR), 0) # ds = AugmentImageComponent(ds, imagenet_augmentors) # ds = td.PrefetchData(ds, 5000, 1) # ds = td.MapDataComponent(ds, lambda x: cv2.imdecode(x, cv2.IMREAD_COLOR), 0) # ds = td.AugmentImageComponent(ds, imagenet_augmentors) # ds = td.PrefetchDataZMQ(ds, num_workers) self.ds = BatchData(ds, batch_size) # self.ds = MultiProcessRunnerZMQ(self.ds, 4) self.ds.reset_state() self.batch_size = batch_size self.num_workers = num_workers self.device = device
def getdata(path, batchsize, isTrain): ds = LMDBSerializer.load(path, shuffle=isTrain) ds = getbatch(ds, batchsize, isTrain) # if isTrain: # ds = MultiProcessRunnerZMQ(ds, 2) return ds
def getdata(path, isTrain): ds = LMDBSerializer.load(path, shuffle=isTrain) ds = getbatch(ds, 64, isTrain) if isTrain: ds = MultiProcessRunnerZMQ(ds, 4) return ds