def __init__(self,
opt,
train='train',
transform=None,
target_transform=None):
self.root = os.path.expanduser(opt['dataroot'])
self.transform = transform
self.target_transform = target_transform
self.train = train # training set or test set
self.create_lmdb = opt['lmdb']
# csv_path = os.path.join(self.root, '{}.csv'.format(train))
#
# data_label_dict = self.parse_csv(csv_path)
# self.data_label_dict = list(
# map(lambda x: (os.path.join(self.root, x[0]), x[1]), data_label_dict))
if self.create_lmdb:
basename = os.path.basename(self.root) + '_train.lmdb'
lmdb_save_path = os.path.join(self.root, basename)
if not os.path.exists(lmdb_save_path):
util.create_lmdb(self.data_label_dict, lmdb_save_path)
self.env, self.data_label_dict = util._get_paths_from_lmdb(
lmdb_save_path)
def __init__(
self,
opt,
train='train',
transform=None,
target_transform=None,
):
self.root = os.path.expanduser(opt['dataroot'])
self.transform = transform
self.target_transform = target_transform
self.train = train # training set or test set
self.create_lmdb = opt['lmdb']
# if train == 'val':
# self.transform.transforms.insert(0, transforms.FiveCrop)
# im_list = glob.glob(os.path.join(self.root, 'jpg')+'/*.jpg')
id_path = os.path.join(self.root, self.id_filename)
data_path = os.path.join(self.root, self.train)
label_dict = {}
with open(id_path, 'r', encoding='utf-8-sig') as f:
im_list = f.readlines()
for line in im_list:
record = line.split(':')
label_dict[record[1]] = int(record[-1])
self.data_label_dict = []
basename = os.path.basename(data_path) + '.lmdb'
lmdb_save_path = os.path.join(self.root, basename)
if (self.create_lmdb and
not os.path.exists(lmdb_save_path)) or not self.create_lmdb:
# now load the image paths
data_folders = os.listdir(data_path)
self.sample_weight = [] # weighted samples for imbalanced dataset
# if self.train != 'test':
for folder in data_folders:
# file = os.path.join(self.root, self.base_folder, f)
path = os.path.join(data_path, folder)
data_images = os.listdir(path)
self.sample_weight.extend([1. / len(data_images)] *
len(data_images))
data_images = [os.path.join(path, p) for p in data_images]
label_id = label_dict[folder]
labels = [label_id - 1] * len(data_images)
self.data_label_dict.extend(list(zip(data_images, labels)))
if self.create_lmdb:
util.create_lmdb(self.data_label_dict,
lmdb_save_path,
multiplier=100)
else:
self.env, self.data_label_dict = util._get_paths_from_lmdb(
lmdb_save_path)
def __init__(self, root, transform=None, target_transform=None, loader=default_loader, resample=0, create_lmdb=False, lmdb_mul=1.): super(ImageFolder, self).__init__(root, loader, IMG_EXTENSIONS, transform=transform, target_transform=target_transform) self.create_lmdb = create_lmdb if self.create_lmdb: basename = os.path.basename(self.root) + '.lmdb' lmdb_save_path = os.path.join(os.path.dirname(self.root), basename) if not os.path.exists(lmdb_save_path): util.create_lmdb(self.samples, lmdb_save_path, multiplier=lmdb_mul) self.env, self.samples = util._get_paths_from_lmdb(lmdb_save_path)
def __init__(self, opt, transform=None):
self.root = os.path.expanduser(opt['dataroot'])
self.transform = transform
self.create_lmdb = False #opt['lmdb']
self.img_list = list_all_files(self.root)
if self.create_lmdb:
basename = os.path.basename(self.root) + '_train.lmdb'
lmdb_save_path = os.path.join(self.root, basename)
if not os.path.exists(lmdb_save_path):
util.create_lmdb(self.img_list, lmdb_save_path)
self.env, self.img_list = util._get_paths_from_lmdb(lmdb_save_path)
def __init__(self, opt, train='train',
transform=None, target_transform=None):
self.root = os.path.expanduser(opt['dataroot'])
self.transform = transform
self.target_transform = target_transform
self.train = train # training set or test set
self.create_lmdb = opt['lmdb']
lmdb_ext = '_{}.lmdb'.format(train)
# if download:
# self.download()
# if not self._check_integrity():
# raise RuntimeError('Dataset not found or corrupted.' +
# ' You can use download=True to download it')
# im_list = glob.glob(os.path.join(self.root, 'jpg')+'/*.jpg')
self.data_path = os.path.join(self.root, self.data_folder)
if self.train == 'train':
label_path = os.path.join(self.root, self.train_label)
# elif self.train == 'val':
# label_path = os.path.join(self.root, self.val_label)
else:
label_path = os.path.join(self.root, self.test_label)
data_label_dict = self.parse_csv(label_path)
self.data_label_dict = list(
map(lambda x: (os.path.join(self.data_path, x[0]+'.jpg'),x[1]), data_label_dict))
if self.create_lmdb:
basename = os.path.basename(self.root) + lmdb_ext
lmdb_save_path = os.path.join(self.root, basename)
if not os.path.exists(lmdb_save_path):
util.create_lmdb(self.data_label_dict, lmdb_save_path)
self.env, self.data_label_dict = util._get_paths_from_lmdb(lmdb_save_path)
def __init__(self, opt, train=True, transform=None, target_transform=None):
def refine_id(img_id):
img_id = str(img_id)
id_len = len(img_id)
for i in range(5 - id_len):
img_id = '0' + img_id
return img_id
self.root = os.path.expanduser(opt['dataroot'])
self.transform = transform
self.target_transform = target_transform
self.train = train # training set or test set
self.create_lmdb = opt['lmdb']
if opt['download']:
self.download()
if not self._check_integrity():
raise RuntimeError('Dataset not found or corrupted.' +
' You can use download=True to download it')
# im_list = glob.glob(os.path.join(self.root, 'jpg')+'/*.jpg')
setid_path = os.path.join(self.root, self.datasplit_filename)
label_path = os.path.join(self.root, self.label_filename)
setid = mat4py.loadmat(setid_path)
labels = mat4py.loadmat(label_path)
# now load the picked numpy arrays
self.data_label_dict = []
if self.train:
for im_id in setid['trnid']:
str_id = refine_id(im_id)
img_file = 'image_{}.jpg'.format(str_id)
img_path = os.path.join(self.root, 'jpg', img_file)
self.data_label_dict.append(
(img_path, int(labels['labels'][im_id - 1]) - 1))
for im_id in setid['valid']:
str_id = refine_id(im_id)
img_file = 'image_{}.jpg'.format(str_id)
img_path = os.path.join(self.root, 'jpg', img_file)
self.data_label_dict.append(
(img_path, int(labels['labels'][im_id - 1]) - 1))
if self.create_lmdb:
basename = os.path.basename(self.root) + '_train.lmdb'
lmdb_save_path = os.path.join(self.root, basename)
if not os.path.exists(lmdb_save_path):
util.create_lmdb(self.data_label_dict,
lmdb_save_path,
multiplier=3)
self.env, self.data_label_dict = util._get_paths_from_lmdb(
lmdb_save_path)
if opt['resample']:
re_index = np.random.randint(0, len(self.train_labels),
len(self.train_labels))
self.train_data = list(
map(lambda x: self.train_data[x], re_index))
self.train_labels = list(
map(lambda x: self.train_labels[x], re_index))
else:
for im_id in setid['tstid']:
str_id = refine_id(im_id)
img_file = 'image_{}.jpg'.format(str_id)
img_path = os.path.join(self.root, 'jpg', img_file)
self.data_label_dict.append(
(img_path, int(labels['labels'][im_id - 1]) - 1))
if self.create_lmdb:
basename = os.path.basename(self.root) + '_val.lmdb'
lmdb_save_path = os.path.join(self.root, basename)
if not os.path.exists(lmdb_save_path):
util.create_lmdb(self.data_label_dict,
lmdb_save_path,
multiplier=6)
self.env, self.data_label_dict = util._get_paths_from_lmdb(
lmdb_save_path)
def __init__(self,
opt,
train='train',
transform=None,
target_transform=None):
self.root = os.path.expanduser(opt['dataroot'])
self.transform = transform
self.target_transform = target_transform
self.train = train # training set or test set
self.create_lmdb = opt['lmdb']
# if opt['download']:
# self.download()
# if not self._check_integrity():
# raise RuntimeError('Dataset not found or corrupted.' +
# ' You can use download=True to download it')
# image_path = os.path.join(self.root, 'images')
# self.data_label_dict = []
# now load the picked numpy arrays
if self.train == 'train':
# trainlabel = os.path.join(self.root, "annotations/trainval.txt")
#
# with open(trainlabel, 'r') as f:
# contents = f.readlines()
# for line in contents:
# details = line.split()
# im_path = os.path.join(image_path, details[0]+'.jpg')
# self.data_label_dict.append((im_path, int(details[1])-1))
if self.create_lmdb:
basename = os.path.basename(self.root) + '_train.lmdb'
lmdb_save_path = os.path.join(self.root, basename)
if not os.path.exists(lmdb_save_path):
util.create_lmdb(self.data_label_dict, lmdb_save_path)
self.env, self.data_label_dict = util._get_paths_from_lmdb(
lmdb_save_path)
# if opt['resample']:
# re_index = np.random.randint(0, len(self.train_labels), len(self.train_labels))
# self.train_data = list(map(lambda x: self.train_data[x], re_index))
# self.train_labels = list(map(lambda x: self.train_labels[x], re_index))
else:
# testlabel = os.path.join(self.root, "annotations/test.txt")
# with open(testlabel, 'r') as f:
# contents = f.readlines()
# for line in contents:
# details = line.split()
# im_path = os.path.join(image_path, details[0]+'.jpg')
# self.data_label_dict.append((im_path, int(details[1])-1))
if self.create_lmdb:
basename = os.path.basename(self.root) + '_test.lmdb'
lmdb_save_path = os.path.join(self.root, basename)
if not os.path.exists(lmdb_save_path):
util.create_lmdb(self.data_label_dict, lmdb_save_path)
self.env, self.data_label_dict = util._get_paths_from_lmdb(
lmdb_save_path)
def __init__(
self,
opt,
train=True,
transform=False,
target_transform=None,
):
"""__init__
:param root:
:param split:
:param is_transform:
:param img_size:
:param augmentations
"""
self.root = os.path.expanduser(opt['dataroot'])
self.transform = transform
self.target_transform = target_transform
self.train = train # training set or test set
self.create_lmdb = opt['lmdb']
self.valid_classes = [0, 1]
self.class_names = ["unlabelled", "building"]
self.class_map = dict(zip(self.valid_classes, range(2)))
# self.img_size = img_size if isinstance(img_size, tuple) else (img_size, img_size)
if self.train:
self.images_base = os.path.join(self.root, "images")
self.annotations_base = os.path.join(self.root, "gt")
lmdb_base = '_val.lmdb'
else:
self.images_base = os.path.join(self.root, "images_val")
self.annotations_base = os.path.join(self.root, "gt_val")
lmdb_base = '_val.lmdb'
image_list = []
for (dirpath, dirnames, filenames) in os.walk(self.images_base):
image_list.extend(filenames)
break
image_list = sorted(image_list)
gt_list = []
for (dirpath, dirnames, filenames) in os.walk(self.annotations_base):
gt_list.extend(filenames)
break
gt_list = sorted(gt_list)
self.data_label_dict = list(zip(image_list, gt_list))
if self.create_lmdb:
basename = os.path.basename(self.root) + lmdb_base
lmdb_save_path = os.path.join(self.root, basename)
if not os.path.exists(lmdb_save_path):
util.create_lmdb(self.data_label_dict,
lmdb_save_path,
is_seg=True)
self.env, self.data_label_dict = util._get_paths_from_lmdb(
lmdb_save_path, is_seg=True)
if self.train and opt['resample']:
re_index = np.random.randint(0, len(self.data_label_dict),
len(self.data_label_dict))
self.data_label_dict = list(
map(lambda x: self.data_label_dict[x], re_index))
def __init__(
self,
opt,
train=True,
transform=None,
target_transform=None,
):
self.root = os.path.expanduser(opt['dataroot'])
self.transform = transform
self.target_transform = target_transform
self.train = train # training set or test set
self.create_lmdb = opt['lmdb']
# if download:
# self.download()
# if not self._check_integrity():
# raise RuntimeError('Dataset not found or corrupted.' +
# ' You can use download=True to download it')
# im_list = glob.glob(os.path.join(self.root, 'jpg')+'/*.jpg')
id_path = os.path.join(self.root, self.id_filename)
label_path = os.path.join(self.root, self.label_filename)
tr_path = os.path.join(self.root, self.tr_filename)
ts_path = os.path.join(self.root, self.ts_filename)
label_mat = io.loadmat(label_path)
tr_mat = io.loadmat(tr_path)
ts_mat = io.loadmat(ts_path)
with open(id_path, 'r') as f:
im_list = f.readlines()
self.data_label_dict = []
# now load the picked numpy arrays
if self.train:
for im_id in tr_mat['tr_idx'][0]:
# file = os.path.join(self.root, self.base_folder, f)
fpath = im_list[im_id - 1].strip('\n').strip('../')
self.data_label_dict.append(
(os.path.join(self.root, fpath),
int(label_mat['labels'][0][im_id - 1]) - 1))
# img = Image.open(os.path.join(self.root, fpath))
# self.train_data.append(img)
# self.train_labels.append(int(label_mat['labels'][0][im_id-1])-1)
if self.create_lmdb:
basename = os.path.basename(self.root) + '_train.lmdb'
lmdb_save_path = os.path.join(self.root, basename)
if not os.path.exists(lmdb_save_path):
util.create_lmdb(self.data_label_dict, lmdb_save_path)
self.env, self.data_label_dict = util._get_paths_from_lmdb(
lmdb_save_path)
if opt['resample']:
re_index = np.random.randint(0, len(self.data_label_dict),
len(self.data_label_dict))
self.data_label_dict = list(
map(lambda x: self.data_label_dict[x], re_index))
else:
for im_id in ts_mat['ts_idx'][0]:
# file = os.path.join(self.root, self.base_folder, f)
fpath = im_list[im_id - 1].strip('\n').strip('../')
self.data_label_dict.append(
(os.path.join(self.root, fpath),
int(label_mat['labels'][0][im_id - 1]) - 1))
# img = Image.open(os.path.join(self.root, fpath))
# self.test_data.append(img)
# self.test_labels.append(int(label_mat['labels'][0][im_id-1])-1)
if self.create_lmdb:
basename = os.path.basename(self.root) + '_val.lmdb'
lmdb_save_path = os.path.join(self.root, basename)
if not os.path.exists(lmdb_save_path):
util.create_lmdb(self.data_label_dict, lmdb_save_path)
self.env, self.data_label_dict = util._get_paths_from_lmdb(
lmdb_save_path)
def __init__(
self,
opt,
train=True,
transform=None,
target_transform=None,
):
self.root = os.path.expanduser(opt['dataroot'])
self.transform = transform
self.target_transform = target_transform
self.train = train # training set or test set
self.create_lmdb = opt['lmdb']
# if download:
# self.download()
# if not self._check_integrity():
# raise RuntimeError('Dataset not found or corrupted.' +
# ' You can use download=True to download it')
self.data_label_dict = []
# now load the picked numpy arrays
if self.train:
label_path = os.path.join(self.root, 'lists',
self.label_train_filename)
label_mat = mat4py.loadmat(label_path)
for i, fpath in enumerate(label_mat['file_list']):
self.data_label_dict.append(
(os.path.join(self.root, 'images/Images', fpath[0]),
int(label_mat['labels'][i][0]) - 1))
if self.create_lmdb:
basename = os.path.basename(self.root) + '_train.lmdb'
lmdb_save_path = os.path.join(self.root, basename)
if not os.path.exists(lmdb_save_path):
util.create_lmdb(self.data_label_dict,
lmdb_save_path,
multiplier=7)
self.env, self.data_label_dict = util._get_paths_from_lmdb(
lmdb_save_path)
if opt['resample']:
re_index = np.random.randint(0, len(self.data_label_dict),
len(self.data_label_dict))
self.data_label_dict = list(
map(lambda x: self.data_label_dict[x], re_index))
else:
label_path = os.path.join(self.root, 'lists',
self.label_test_filename)
label_mat = mat4py.loadmat(label_path)
for i, fpath in enumerate(label_mat['file_list']):
self.data_label_dict.append(
(os.path.join(self.root, 'images/Images', fpath[0]),
int(label_mat['labels'][i][0]) - 1))
if self.create_lmdb:
basename = os.path.basename(self.root) + '_val.lmdb'
lmdb_save_path = os.path.join(self.root, basename)
if not os.path.exists(lmdb_save_path):
util.create_lmdb(self.data_label_dict,
lmdb_save_path,
multiplier=6)
self.env, self.data_label_dict = util._get_paths_from_lmdb(
lmdb_save_path)
def __init__(self, opt, train=True, transform=None, target_transform=None):
self.root = os.path.expanduser(opt['dataroot'])
self.root = os.path.join(self.root, 'data')
self.transform = transform
self.target_transform = target_transform
self.train = train # training set or test set
self.create_lmdb = opt['lmdb']
# if download:
# self.download()
# if not self._check_integrity():
# raise RuntimeError('Dataset not found or corrupted.' +
# ' You can use download=True to download it')
variants_path = os.path.join(self.root, self.variants_filename)
numeric_label_dict = {}
with open(variants_path, 'r') as f:
content = f.readlines()
for i, c in enumerate(content):
numeric_label_dict[c.strip()] = i
self.data_label_dict = []
# now load the picked numpy arrays
if self.train:
label_path = os.path.join(self.root, self.label_train_filename)
with open(label_path, 'r') as f:
content = f.readlines()
for line in content:
details = line.split()
fpath = details[0] + '.jpg'
label_name = ' '.join(details[1:])
self.data_label_dict.append(
(os.path.join(self.root, 'images',
fpath), numeric_label_dict[label_name]))
if self.create_lmdb:
basename = os.path.basename(self.root) + '_train.lmdb'
lmdb_save_path = os.path.join(self.root, basename)
if not os.path.exists(lmdb_save_path):
util.create_lmdb(self.data_label_dict, lmdb_save_path)
self.env, self.data_label_dict = util._get_paths_from_lmdb(
lmdb_save_path)
if opt['resample']:
re_index = np.random.randint(0, len(self.data_label_dict),
len(self.data_label_dict))
self.data_label_dict = list(
map(lambda x: self.data_label_dict[x], re_index))
else:
label_path = os.path.join(self.root, self.label_test_filename)
with open(label_path, 'r') as f:
content = f.readlines()
for line in content:
details = line.split()
fpath = details[0] + '.jpg'
label_name = ' '.join(details[1:])
self.data_label_dict.append(
(os.path.join(self.root, 'images',
fpath), numeric_label_dict[label_name]))
if self.create_lmdb:
basename = os.path.basename(self.root) + '_val.lmdb'
lmdb_save_path = os.path.join(self.root, basename)
if not os.path.exists(lmdb_save_path):
util.create_lmdb(self.data_label_dict, lmdb_save_path)
self.env, self.data_label_dict = util._get_paths_from_lmdb(
lmdb_save_path)