def load_imagenet_val(path_img, path_info, args):
img_name, img_label = load_header_imagenet(load_file(path_val_info))
chunks_name = [
img_name[x:x + args.val_size]
for x in range(0, len(img_name), args.val_size)
]
chunks_label = [
img_label[x:x + args.val_size]
for x in range(0, len(img_label), args.val_size)
]
for i, (c_n, c_l) in enumerate(zip(chunks_name, chunks_label)):
if os.path.exists('./dataset/%s_%s_val_%i.p' %
(args.d, args.model, i)):
print('File exists in your directory')
continue
else:
x_val, y_val = list(), list()
for n, l in zip(c_n, c_l):
img = load_img_imagenet(path_img + n, args)
if len(img.shape) == 4:
x_val.append(img)
y_val.append(l)
x_val = np.concatenate(x_val, axis=0)
y_val = np.array(y_val)
print(x_val.shape, y_val.shape)
pickle.dump(
(x_val, y_val),
open('./dataset/%s_%s_val_%i.p' % (args.d, args.model, i),
'wb'),
protocol=4)
print('Random validation %i-th' % (i))
print('Now you can load the validation dataset')
exit()
def process_batch(path_imagenet_folder, data):
image_path, label = load_header_imagenet(data)
x_batch = list()
for p in image_path:
x = load_img_imagenet(path_imagenet_folder + p)
x = x.reshape(-1, 3, 224, 224)
x_batch.append(x)
x_batch = np.concatenate(x_batch)
y_batch = np.asarray(label)
return (x_batch, y_batch)
def generated_batch_image(data, args):
data_path, data_label = list(), list()
for d in data:
path_img, label_img = d.split()[0], int(d.split()[1])
img_shape = load_img_imagenet(path_img, args)
if len(img_shape.shape) == 4:
data_path.append(img_shape)
data_label.append(label_img)
if len(data_path) > 0:
data_path = np.concatenate(data_path, axis=0)
data_label = np.array(data_label)
data_path = np.reshape(data_path, (-1, 3, args.image_size, args.image_size))
return (data_path, data_label, True)
else:
return (data_path, data_label, False)
def load_imagenet_random_train_ver2(path_img, path_info, args):
"""Load IMAGENET dataset random training for calculating DSA or LSA
Args:
path_img (string): Folder of train dataset
path_info (string): A file which includes the information of training data
args: Keyboards arguments
"""
imgnet_info = load_file(path_info)
name_folders = [p.split('/')[0].strip() for p in imgnet_info]
name_folders = list(sorted(set(name_folders)))
random_train_file = list()
random.seed(0)
for i, n in enumerate(name_folders):
random_train_file, x_random_train, y_random_train = list(), list(
), list()
random_name_file = sorted(
random.sample(os.listdir(path_img + n), args.random_train_size))
process_folder = [
load_img_imagenet(path_img + n + '/' + r, args)
for r in random_name_file
]
zip_process = [(p, r) for p, r in zip(process_folder, random_name_file)
if len(p.shape) == 4]
process_folder = [p for (p, r) in zip_process]
random_name_file = [r for (p, r) in zip_process]
if len(process_folder) > 0:
process_folder = np.concatenate(process_folder, axis=0)
label_folder = get_label_imagenet(name_folder=n,
imagnet_info=imgnet_info)
label_folder = np.array(
[label_folder for i in range(len(process_folder))])
print(len(random_name_file), process_folder.shape, label_folder.shape)
print('Random training the folder %i-th which has name %s' % (i, n))
pickle.dump((random_name_file, process_folder, label_folder),
open(
'./dataset_imagenet/%s_%s_random_train_%i.p' %
(args.d, args.model, i), 'wb'),
protocol=4)
print('Now you can load the training dataset')
exit()
def load_imagenet_random_train(path_img, path_info, args):
"""Load IMAGENET dataset random training for calculating DSA or LSA
Args:
path_img (string): Folder of train dataset
path_info (string): A file which includes the information of training data
args: Keyboards arguments
Returns:
x_train (array), y_train (array): Numpy array for input and label
"""
imgnet_info = load_file(path_info)
name_folders = [p.split('/')[0].strip() for p in imgnet_info]
name_folders = list(sorted(set(name_folders)))
random_train_file = list()
for m in range(args.random_train_num_start, args.random_train_num_end):
random.seed(m)
# for i, n in enumerate(name_folders):
# random_name_file = sorted(random.sample(os.listdir(path_img + n), args.random_train_size))
# process_folder = [load_img_imagenet(path_img + n + '/' + r, args) for r in random_name_file]
# process_folder = [p for p in process_folder if len(p.shape) == 4]
# if len(process_folder) > 0:
# random_train_file += random_name_file
# print(m, i)
if os.path.exists('./dataset/%s_%s_random_train_%i.p' %
(args.d, args.model, m)):
print('File exists in your directory')
else:
x_random_train, y_random_train = list(), list()
for i, n in enumerate(name_folders):
random_name_file = sorted(
random.sample(os.listdir(path_img + n),
args.random_train_size))
process_folder = [
load_img_imagenet(path_img + n + '/' + r, args)
for r in random_name_file
]
process_folder = [
p for p in process_folder if len(p.shape) == 4
]
if len(process_folder) > 0:
process_folder = np.concatenate(process_folder, axis=0)
label_folder = get_label_imagenet(name_folder=n,
imagnet_info=imgnet_info)
label_folder = np.array(
[label_folder for i in range(args.random_train_size)])
x_random_train.append(process_folder)
y_random_train.append(label_folder)
print(
'Random training %i-th of the folder %i-th which has name %s'
% (m, i, n))
x_random_train = np.concatenate(x_random_train, axis=0)
y_random_train = np.concatenate(y_random_train, axis=0)
pickle.dump((x_random_train, y_random_train),
open(
'./dataset/%s_%s_random_train_%i.p' %
(args.d, args.model, m), 'wb'),
protocol=4)
write_file(
'./dataset/%s_%s_random_train_name_file.txt' % (args.d, args.model),
random_train_file)
print('Now you can load the training dataset')
exit()