def generate_patch(dataset_path=DATASET_PATH): """ Generate patches in the dataset. :return: """ image_pre = ImagePreprocessor(base_dir=dataset_path, data_dir=os.path.join(dataset_path, 'data')) image_pre.generate_patches()
def dump(patch_size, stride, cut_img_threshold, expected_max_patches_per_img): import cv2 _data = ImagePreprocessor(base_dir=config.base_dataset_dir).get_dataset_full(data_selection='sup', label_type='non-str') log_file = open(config.dataset_dir/"log.txt", 'w') tic = time.time() print(f"Config: patch_size={patch_size}x{patch_size} stride={stride}x{stride} threshold={cut_img_threshold}", file=log_file) def get_sample(imgs): patches, positions, demos = zip(*[cut_img(img, width=patch_size, height=patch_size, xstep=stride, ystep=stride, threshold=cut_img_threshold, write_mode=False, output_folder=None ) for img in imgs]) patches = np.vstack([np.stack(p) for p in patches if len(p) > 0]) # dropout excessive patches to keep the size of dateset reasonable if len(patches) > expected_max_patches_per_img: random_choice = np.zeros(len(patches), dtype=np.bool) random_choice[:expected_max_patches_per_img] = True np.random.shuffle(random_choice) patches = patches[random_choice] # rate = 1 - np.exp(-expected_max_patches_per_img / len(patches)) # random_choice = np.random.random(len(patches)) < rate patches = np.mean(patches.astype(np.float32), axis=-1) # convert to gray scale return patches, demos for ind, (label1, label2, imgs) in enumerate(_data): labels = set(l for l in label1 + label2 if l < config.n_labels) label_str = ''.join(['1' if i in labels else '0' for i in range(config.n_labels)]) patches, demos = get_sample(imgs) if patches is None: continue with open(config.dataset_dir/f"sample{ind:04d}_{label_str}", 'bw') as f: pickle.dump({'patches': patches, 'labels': labels}, f) for i, demo in enumerate(demos): cv2.imwrite(config.dataset_dir/"demo"/f"sample{ind:04d}-{i}.jpg", demo) print(f"Dumping sample{ind:04d} with patches: {patches.shape}, labels: {labels}", file=log_file) if ind % 10 == 9: toc = time.time() print(f"Speed: {toc - tic:.3f} sec / {ind + 1} samples = {(toc - tic) / (ind + 1):.3f} sec/samples", file=log_file) log_file.flush() tac = time.time() log_file.close()
def data_construction_v2(target_labels, batch_size, ratio=8 / 9): image_pre = ImagePreprocessor(base_dir=DATASET_PATH) l1, l2, d = image_pre.get_dataset_patched(size=20, data_selection='all', label_type='int', exist='new') batch_num = int(500 * 270 / batch_size) # batch_num=5400 if batch_size=25 r, l = [[] for _ in range(int(batch_num * 8 / 9))], [[] for _ in range(int(batch_num * 8 / 9))] tr, tl = [[] for _ in range(int(batch_num / 9))], [[] for _ in range(int(batch_num / 9))] label_counter = {target: 0 for target in target_labels} for i in range(len(l1)): lbs = set(l1[i] + l2[i]) sz = len(d[i]) for target in target_labels: if target in lbs: label_counter[target] += 1 for j in range(sz): if j < int(sz * ratio): base = 0 if i < int(len(l1) / 2) else int(batch_num * 4 / 9) r[j + base].append(d[i][j]) label_array = [] for target in target_labels: label_array.append([1, 0] if target in lbs else [0, 1]) l[j + base].append(label_array) else: base = 0 if i < int(len(l1) / 2) else int(batch_num / 18) tr[j + base - int(sz * ratio)].append(d[i][j]) label_array = [] for target in target_labels: label_array.append([1, 0] if target in lbs else [0, 1]) tl[j + base - int(sz * ratio)].append(label_array) raws, labels = [np.array(x) for x in r], [np.array(x) for x in l] test_raws, test_labels = [np.array(x) for x in tr], [np.array(x) for x in tl] loss_array = [[0.5 / (label_counter[label] / len(l1)), 0.5 / (1 - label_counter[label] / len(l1))] for label in label_counter.keys()] print("Dataset constructed with batch size {}".format(batch_size)) return raws, labels, test_raws, test_labels, loss_array
def __init__(self, raws, labels, test_raws, test_labels, keep_pb=0.5, epoch_size=100, learning_rate=0.001, start_step=0, loss_array=None, detail_log=False, open_summary=False, new_ckpt_internal=0, network_mode=None): """ Convolutional neural network. Before running, you should modify the configuration first, which is in 'config.py'. :param raws: list Raw data in training set. :param labels: list Labels in training set. :param test_raws: list Raw data in test set. :param test_labels: list Labels in test set. :param keep_pb: float The keep probabilities used in dropout layers. :param epoch_size: int Epoch size :param learning_rate: float Learning rate :param start_step: int Current start step, which is used in further training based on existed model to prevent the disorder of current epoch. :param detail_log: boolean Whether detailed log is outputed. If detailed log mode is on, the infomation of each batch will also be printed. :param open_summary: boolean Whether summary is open. If summary mode is on, the summary graph and logs will be written into the log file, which can be shown in tensorboard. :param new_ckpt_internal: int The epoch internal for new checkpoint file generation. If set -1, the checkpoint file will not be saved. If set 0, there will only exist one checkpoint file through the whole training. If set n (n>0), every n step will generate a new checkpoint file. For example, when n=5, epoch size=100, then 20 checkpoint files will be created all together. :param network_mode: string The network type in training. If set None(default), use the default Lenet binary relevance classifiers. If set 'chain', use the Lenet classifier chain. """ self._raws = raws self._labels = labels self._test_raws = test_raws self._test_labels = test_labels self._keep_pb = keep_pb self._epoch_size = epoch_size self._start_step = start_step self._learning_rate = learning_rate self._detail_log = detail_log self._open_summary = open_summary self._new_ckpt_internal = new_ckpt_internal self._network_mode = network_mode self._image_pre = ImagePreprocessor(base_dir=DATASET_PATH) [_, self._input_width, self._input_height, self._input_channels] = raws[0].shape [_, self._label_nums, self._classes] = labels[0].shape self._loss_array = [[0.5, 0.5] for _ in range(self._labels)] if loss_array is None else loss_array self._x = tf.placeholder(tf.float32, shape=[None, self._input_width, self._input_height, self._input_channels], name="input_x") self._y = tf.placeholder(tf.float32, shape=[None, self._label_nums, self._classes], name="input_y") self._keep_prob = tf.placeholder(tf.float32, name="keep_prob") self._is_training = tf.placeholder(tf.bool, name="is_training") self._global_step = tf.Variable(0, trainable=False)
def data_construction(target_labels, ratio=8 / 9): """ We separate the 500 data folders into 2 groups, 0-250 and 250-500. Since we have 270(180) photos each folder, we can cut the dataset into 540 batches. Take the default train-test ratio 8/9 as an examle. The train-set size is 480 and test-set size is 60. :param target_labels: list, [label number] The labels needed in the training. :param ratio: float, default 0.8888888 The ratio of training set over test set. :return: raws, labels, test_raws, test_labels, loss_array raws: list, [training set size * photo numbers * photo width * photo height * channels], default [480 * 250 * 20 * 20 * 3] Training set data. labels: list, [training set size * photo numbers * label number * 2], default [480 * 250 * 6 * 2] Training set labels. The third dimension is the one hot encoding, so that [1, 0] means inclusion and [0, 1] means exclusion. test_raws: list, [test set size * photo numbers * photo width * photo height * channels], default [60 * 250 * 20 * 20 * 3] Test set data. test_labels: list, [test set size * photo numbers * label number * 2], default [60 * 250 * 6 * 2] Test set labels. The shape is the same to @labels. loss_array: list, [label number * 2], default [6 * 2] This is the weight loss matrix, which is used in weighted loss function. The formulation of loss array is: [0.5 / (label counter / label size), 0.5 / (1 - label counter / label size)] Loss array provide a balance in training for data under non-uniform distribution. """ image_pre = ImagePreprocessor(base_dir=DATASET_PATH, data_dir=os.path.join(DATASET_PATH, 'data')) l1, l2, d = image_pre.get_dataset_patched(size=20, data_selection='all', label_type='int', exist='new') r, l = [[] for _ in range(480)], [[] for _ in range(480)] tr, tl = [[] for _ in range(60)], [[] for _ in range(60)] label_counter = {target: 0 for target in target_labels} for i in range(len(l1)): lbs = set(l1[i] + l2[i]) sz = len(d[i]) for target in target_labels: if target in lbs: label_counter[target] += 1 for j in range(sz): if j < int(sz * ratio): base = 0 if i < int(len(l1) / 2) else 240 r[j + base].append(d[i][j]) label_array = [] for target in target_labels: label_array.append([1, 0] if target in lbs else [0, 1]) l[j + base].append(label_array) else: base = 0 if i < int(len(l1) / 2) else 30 tr[j + base - int(sz * ratio)].append(d[i][j]) label_array = [] for target in target_labels: label_array.append([1, 0] if target in lbs else [0, 1]) tl[j + base - int(sz * ratio)].append(label_array) raws, labels = [np.array(x) for x in r], [np.array(x) for x in l] test_raws, test_labels = [np.array(x) for x in tr], [np.array(x) for x in tl] loss_array = [[0.5 / (label_counter[label] / len(l1)), 0.5 / (1 - label_counter[label] / len(l1))] for label in label_counter.keys()] print("Dataset constructed") return raws, labels, test_raws, test_labels, loss_array
def get_data(dataset_path): image_pre = ImagePreprocessor(base_dir=dataset_path, data_dir=os.path.join(dataset_path, 'testdata')) vals, data = image_pre.get_valset_patched(exist='new') data = [piece[-30:] for piece in data] return vals, data
def get_data_trainset(dataset_path): image_pre = ImagePreprocessor(base_dir=dataset_path, data_dir=os.path.join(dataset_path, 'testdata')) l1, l2, d = image_pre.get_dataset_patched(size=20, data_selection='all', label_type='int', exist='new') return l1, l2, d