def sequential_synthetic_dataset(root_path, dataset_name, data_type):
"""Generate a synthetic dataset for regression."""
if data_type == "dense":
X, y = make_classification(
n_samples=10000,
n_features=100,
n_informative=90,
n_classes=2,
random_state=42,
)
else:
raise NotImplementedError(
"{} synthetic dataset is " "not supported.".format(data_type)
)
data = LIBSVMDataset(X, y, False)
lmdb_file_path = os.path.join(
root_path, "{}_{}.lmdb".format(dataset_name, data_type)
)
ds1 = PrefetchDataZMQ(
data,
)
LMDBSerializer.save(ds1, lmdb_file_path)
print("Dumped dataflow to {} for {}".format(lmdb_file_path, dataset_name))
def sequential_epsilon_or_rcv1(root_path, name, data_type, is_sparse):
data = LIBSVMDataset(root_path, name, data_type, is_sparse)
lmdb_file_path = join(root_path, '{}_{}.lmdb'.format(name, data_type))
print('dump_dataflow_to_lmdb for {}'.format(lmdb_file_path))
ds1 = PrefetchDataZMQ(data, nr_proc=1)
LMDBSerializer.save(ds1, lmdb_file_path)
def sequential_epsilon_or_rcv1(root_path, name, data_type):
features, labels, is_sparse = _load_libsvm_data(root_path, name, data_type)
data = LIBSVMDataset(features, labels, is_sparse)
lmdb_file_path = os.path.join(root_path, "{}_{}.lmdb".format(name, data_type))
ds1 = PrefetchDataZMQ(data)
LMDBSerializer.save(ds1, lmdb_file_path)
print("Dumped dataflow to {} for {}".format(lmdb_file_path, name))
def sequential_downsampled_imagenet(args):
data = DownsampledImageNet(args.data_dir, args.data_type, args.img_size)
lmdb_file_path = os.path.join(
args.data_dir, f"imagenet{args.img_size}_{args.data_type}.lmdb")
# delete file if exists.
if os.path.exists(lmdb_file_path) and args.force_delete == 1:
os.remove(lmdb_file_path)
# serialize to the target path.
ds1 = PrefetchDataZMQ(data, num_proc=1)
LMDBSerializer.save(ds1, lmdb_file_path)
def sequential_synthetic_dataset(root_path, dataset_name):
"""Generate a synthetic dataset for regression."""
if 'dense' in dataset_name:
X, y = make_classification(n_samples=10000, n_features=100, n_informative=90, n_classes=2, random_state=42)
else:
raise NotImplementedError("{} synthetic dataset is not supported.".format(dataset_name))
data = SyntheticLIBSVMDataset(X, y)
lmdb_file_path = join(root_path, '{}.lmdb'.format(dataset_name))
print('dump_dataflow_to_lmdb for {}'.format(lmdb_file_path))
ds1 = PrefetchDataZMQ(data, nr_proc=1)
LMDBSerializer.save(ds1, lmdb_file_path)
def compute_mean_std(db, fname):
ds = LMDBSerializer.load(db, shuffle=False)
ds.reset_state()
o = OnlineMoments()
for dp in get_tqdm(ds):
feat = dp[0] # len x dim
for f in feat:
o.feed(f)
logger.info("Writing to {} ...".format(fname))
with open(fname, 'wb') as f:
f.write(serialize.dumps([o.mean, o.std]))
def compute_mean_std(db, fname):
ds = LMDBSerializer.load(db, shuffle=False)
ds.reset_state()
o = OnlineMoments()
for dp in get_tqdm(ds):
feat = dp[0] # len x dim
for f in feat:
o.feed(f)
logger.info("Writing to {} ...".format(fname))
with open(fname, 'wb') as f:
f.write(serialize.dumps([o.mean, o.std]))
def getdata(path, isTrain):
ds = LMDBSerializer.load(path, shuffle=isTrain)
# Graph Benchmark
# ds=FakeData([[10,10],[10,10],[10,10],[10,10],[10],[10],[10,10],[1],[1],[1]], 1000, random=False,dtype=['int32', 'int32', 'int32', 'int32', 'int32', 'int32',\
# 'int32', 'int32', 'int32', 'int32'], domain=[(0, 100), (0, 120),(0,120),(0,1),(0,100),(0,100),(0,100),(0,52),(0,115),(0,115)])
ds = getbatch(ds, 32, isTrain)
if isTrain:
ds = MultiProcessRunnerZMQ(ds, 4)
return ds
def __init__(self, file_location, batch_size, train=True, shuffle=True, full=False, batch_from_disk=150):
self.batch_size = batch_size
self.train = train
if train:
self.ds = MyLMDBSerializer.load(file_location, shuffle=shuffle, batch_from_disk=batch_from_disk)
self.ds = MyLocallyShuffleData(self.ds, buffer_size=10000, shuffle_interval=500)
self.ds = MultiProcessRunnerZMQ(self.ds, num_proc=1, hwm=10000)
self.len_ = 1281167
else:
self.ds = LMDBSerializer.load(file_location, shuffle=False)
self.ds = MultiProcessRunnerZMQ(self.ds, num_proc=1, hwm=10000)
self.len_ = 50000
self.ds.reset_state()
self.batches_in_epoch = int(math.ceil(self.len_ / self.batch_size))
def __init__(self,
mode,
batch_size=256,
shuffle=False,
num_workers=25,
cache=50000,
device='cuda'):
# enumerate standard imagenet augmentors
imagenet_augmentors = fbresnet_augmentor(mode == 'train')
# load the lmdb if we can find it
base_dir = '/userhome/cs/u3003679/'
lmdb_loc = os.path.join(base_dir, 'ILSVRC-{}.lmdb'.format(mode))
#lmdb_loc = os.path.join(os.environ['IMAGENET'], 'ILSVRC-%s.lmdb'%mode)
ds = LMDBSerializer.load(lmdb_loc, shuffle=shuffle)
ds = LocallyShuffleData(ds, cache)
# ds = td.LMDBDataPoint(ds)
def f(dp):
x, label = dp
x = cv2.imdecode(x, cv2.IMREAD_COLOR)
for aug in imagenet_augmentors:
x = aug.augment(x)
return x, label
ds = MultiProcessMapDataZMQ(ds, num_proc=8, map_func=f)
# ds = MapDataComponent(ds, lambda x: cv2.imdecode(x, cv2.IMREAD_COLOR), 0)
# ds = AugmentImageComponent(ds, imagenet_augmentors)
# ds = td.PrefetchData(ds, 5000, 1)
# ds = td.MapDataComponent(ds, lambda x: cv2.imdecode(x, cv2.IMREAD_COLOR), 0)
# ds = td.AugmentImageComponent(ds, imagenet_augmentors)
# ds = td.PrefetchDataZMQ(ds, num_workers)
self.ds = BatchData(ds, batch_size)
# self.ds = MultiProcessRunnerZMQ(self.ds, 4)
self.ds.reset_state()
self.batch_size = batch_size
self.num_workers = num_workers
self.device = device
def getdata(path, isTrain):
ds = LMDBSerializer.load(path, shuffle=isTrain)
ds = getbatch(ds, 64, isTrain)
if isTrain:
ds = MultiProcessRunnerZMQ(ds, 4)
return ds
TailLabel,
]
yield output
if __name__ == "__main__":
parser = argparse.ArgumentParser()
subparsers = parser.add_subparsers(title="command", dest="command")
parser_db = subparsers.add_parser("build",
help="build train/test database")
parser_db.add_argument("--dataset",
help="path to train/test data",
required=True)
parser_db.add_argument("--db", help="output lmdb file", required=True)
parser_eval = subparsers.add_parser("eval", help="bulid p@n eval database")
parser_eval.add_argument("--dataset",
help="path to eval data",
required=True)
parser_eval.add_argument("--db",
help="output eval lmdb file",
required=True)
args = parser.parse_args()
if args.command == "build":
data = pickle.load(open(args.dataset, "rb"))
ds = Raw(data)
LMDBSerializer.save(ds, args.db)
elif args.command == "eval":
data = pickle.load(open(args.dataset, "rb"))
ds = Raw(data)
LMDBSerializer.save(ds, args.db)
ds.reset_state()
o = OnlineMoments()
for dp in get_tqdm(ds):
feat = dp[0] # len x dim
for f in feat:
o.feed(f)
logger.info("Writing to {} ...".format(fname))
with open(fname, 'wb') as f:
f.write(serialize.dumps([o.mean, o.std]))
if __name__ == '__main__':
parser = argparse.ArgumentParser()
subparsers = parser.add_subparsers(title='command', dest='command')
parser_db = subparsers.add_parser('build', help='build a LMDB database')
parser_db.add_argument('--dataset',
help='path to TIMIT TRAIN or TEST directory', required=True)
parser_db.add_argument('--db', help='output lmdb file', required=True)
parser_stat = subparsers.add_parser('stat', help='compute statistics (mean/std) of dataset')
parser_stat.add_argument('--db', help='input lmdb file', required=True)
parser_stat.add_argument('-o', '--output',
help='output statistics file', default='stats.data')
args = parser.parse_args()
if args.command == 'build':
ds = RawTIMIT(args.dataset)
LMDBSerializer.save(ds, args.db)
elif args.command == 'stat':
compute_mean_std(args.db, args.output)
'char': sentiment_c,
'num': m
}, open('./data/sentiment.pkl', 'wb'))
# make train/dev/test
for i in range(10):
train_ori = get_train(data, 10, i)
test_ori = get_test(data, 10, i)
train = []
dev = []
test = []
for j in range(2):
random.shuffle(train_ori[j])
x = len(train_ori[j]) * 9 // 10
train += train_ori[j][:x]
dev += train_ori[j][x:]
test += test_ori
random.shuffle(train)
random.shuffle(dev)
random.shuffle(test)
train_ = process_data(train, word2id, char2id)
dev_ = process_data(dev, word2id, char2id)
test_ = process_data(test, word2id, char2id)
train_data = MEANdata(train_)
dev_data = MEANdata(dev_)
test_data = MEANdata(test_)
os.system('mkdir mdb%s' % i)
LMDBSerializer.save(train_data, './mdb{}/train.mdb'.format(i))
LMDBSerializer.save(dev_data, './mdb{}/dev.mdb'.format(i))
LMDBSerializer.save(test_data, './mdb{}/test.mdb'.format(i))
# --------------------
parser.add_argument('--imagenet_folder')
parser.add_argument('--val', action='store_true')
parser.add_argument('--train', action='store_true')
parser.add_argument('--lmdb_file', type=str)
args = parser.parse_args()
if args.val and args.train:
print(
"Train and Validation options are mutually exclusive! Chose only one."
)
if args.val:
print(
"We are generating the lmdb file containing validation images of imagenet."
)
print(f"The file will be saved at {args.lmdb_file}.lmdb")
ds0 = BinaryILSVRC12(os.path.expanduser(args.imagenet_folder), 'val')
ds1 = MultiProcessRunnerZMQ(ds0, num_proc=1)
LMDBSerializer.save(ds1, f"{os.path.expanduser(args.lmdb_file)}.lmdb")
elif args.train:
print(
"We are generating the lmdb file containing training images of imagenet."
)
print(f"The file will be saved at {args.lmdb_file}.lmdb")
ds0 = BinaryILSVRC12(os.path.expanduser(args.imagenet_folder), 'train')
ds1 = MultiProcessRunnerZMQ(ds0, num_proc=1)
LMDBSerializer.save(ds1, f"{os.path.expanduser(args.lmdb_file)}.lmdb")
self.count = 0
def __len__(self):
return len(self.data)
def __iter__(self):
for bag in data:
# X = bag['X']
# Pos1 = bag['Pos1']
# Pos2 = bag['Pos2']
# DepMask = bag['DepMask']
HeadPos = bag["HeadPos"]
TailPos = bag["TailPos"]
if max(max(bag["HeadPos"]), max(bag["TailPos"])) > 100:
self.count += 1
# DepLabel = bag['Dep']
# ReLabel = bag['Y']
# HeadLabel = bag['HeadLabel']
# TailLabel = bag['TailLabel']
# output = [X, Pos1, Pos2, DepMask, HeadPos, TailPos, DepLabel, ReLabel, HeadLabel, TailLabel]
output = [HeadPos, TailPos]
yield output
if __name__ == "__main__":
data = pickle.load(open("/data/PKL/train.pkl", "rb"))
ds = Raw(data)
LMDBSerializer.save(ds, "/data/MLRE/testpkl")
print(ds.count)
def getdata(path, batchsize, isTrain):
ds = LMDBSerializer.load(path, shuffle=isTrain)
ds = getbatch(ds, batchsize, isTrain)
# if isTrain:
# ds = MultiProcessRunnerZMQ(ds, 2)
return ds
parser.add_argument('database_dir',
type=str,
default=None,
help='location to save output database')
args = parser.parse_args()
class BinaryILSVRC12(dataset.ILSVRC12Files):
def get_data(self):
for fname, label in super(BinaryILSVRC12, self).__iter__():
with open(fname, 'rb') as f:
jpeg = f.read()
jpeg = np.asarray(bytearray(jpeg), dtype='uint8')
yield [jpeg, label]
if args.database_dir is None:
lmdb_path = args.imagenet
else:
lmdb_path = args.database_dir
os.environ['TENSORPACK_DATASET'] = os.path.join(lmdb_path,
"tensorpack_data")
if not os.path.exists(os.environ['TENSORPACK_DATASET']):
os.mkdir(os.environ['TENSORPACK_DATASET'])
for name in ['train', 'val']:
db_filename = 'ILSVRC-%s.lmdb' % name
db_loc = os.path.join(lmdb_path, db_filename)
print(f"Processing {args.imagenet} {name} to {db_loc}...")
ds0 = BinaryILSVRC12(args.imagenet, name)
ds1 = MultiProcessRunnerZMQ(ds0, num_proc=1)
LMDBSerializer.save(ds1, db_loc)