def codes_metadata_generator(params, data, metadataStats, codesStats): """ Custom generator that yields a vector containign the 4096-d CNN codes output by ResNet50 and metadata features (if params set to use). :param params: global parameters, used to find location of the dataset and json file :param data: list of objects containing the category labels and paths to CNN codes and images :param metadataStats: metadata stats used to normalize metadata features :yield (codesMetadata,labels): 4096-d CNN codes + metadata features (if set), and labels (categorical form) """ N = len(data) idx = np.random.permutation(N) batchInds = get_batch_inds(params.batch_size_lstm, idx, N) trainKeys = list(data.keys()) executor = ProcessPoolExecutor(max_workers=params.num_workers) while True: for inds in batchInds: batchKeys = [trainKeys[ind] for ind in inds] codesMetadata, labels = load_lstm_batch(params, data, batchKeys, metadataStats, codesStats, executor) yield (codesMetadata, labels)
def img_metadata_generator(params, data, metadataStats): """ Custom generator that yields images or (image,metadata) batches and their category labels (categorical format). :param params: global parameters, used to find location of the dataset and json file :param data: list of objects containing the category labels and paths to images and metadata features :param metadataStats: metadata stats used to normalize metadata features :yield (imgdata,labels) or (imgdata,metadata,labels): image data, metadata (if params set to use), and labels (categorical form) """ N = len(data) idx = np.random.permutation(N) batchInds = get_batch_inds(params.batch_size_cnn, idx, N) executor = ProcessPoolExecutor(max_workers=params.num_workers) while True: for inds in batchInds: batchData = [data[ind] for ind in inds] imgdata, metadata, labels = load_cnn_batch(params, batchData, metadataStats, executor) if params.use_metadata: yield ([imgdata, metadata], labels) else: yield (imgdata, labels)
def img_metadata_generator(params, data, metadataStats): """ Custom generator that yields images or (image,metadata) batches and their category labels (categorical format). :param params: global parameters, used to find location of the dataset and json file :param data: list of objects containing the category labels and paths to images and metadata features :param metadataStats: metadata stats used to normalize metadata features :yield (imgdata,labels) or (imgdata,metadata,labels): image data, metadata (if params set to use), and labels (categorical form) """ N = len(data) idx = np.random.permutation(N) batchInds = get_batch_inds(params.batch_size_cnn, idx, N) executor = ProcessPoolExecutor(max_workers=params.num_workers) while True: for inds in batchInds: batchData = [data[ind] for ind in inds] imgdata, metadata, labels = load_cnn_batch(params, batchData, metadataStats, executor) if (params.generator == 'flip'): datagen = ImageDataGenerator(horizontal_flip=True, vertical_flip=True) elif (params.generator == 'zoom'): datagen = ImageDataGenerator(zoom_range=[0.9, 1.0], horizontal_flip=True, vertical_flip=True) elif (params.generator == 'shift'): datagen = ImageDataGenerator(width_shift_range=0.2, height_shift_range=0.2, horizontal_flip=True, vertical_flip=True) #datagen.fit(imgdata) #batches = datagen.flow(imgdata, labels, batch_size=params.batch_size_cnn, shuffle=False, save_to_dir='output', save_prefix='aug', save_format='jpg') batches = datagen.flow(imgdata, labels, batch_size=params.batch_size_cnn, shuffle=False) idx0 = 0 for batch in batches: #print (idx) idx1 = idx0 + batch[0].shape[0] if params.use_metadata: #yield ([imgdata, metadata], labels) yield ([batch[0], metadata[idx0:idx1]], batch[1]) else: #yield (imgdata, labels) yield (batch[0], batch[1]) idx0 = idx1 if idx1 >= imgdata.shape[0]: break
def codes_metadata_generator(params, data, metadataStats, codesStats, class_aware_sampling = True, temporal_dropout = True): """ Custom generator that yields a vector containign the 4096-d CNN codes output by ResNet50 and metadata features (if params set to use). :param params: global parameters, used to find location of the dataset and json file :param data: list of objects containing the category labels and paths to CNN codes and images :param metadataStats: metadata stats used to normalize metadata features :yield (codesMetadata,labels): 4096-d CNN codes + metadata features (if set), and labels (categorical form) """ N = len(data) if class_aware_sampling: data_labels = [datum['category'] for datum in data.values()] label_to_idx = defaultdict(list) for i, label in enumerate(data_labels): label_to_idx[label].append(i) running_label_to_idx = copy.deepcopy(label_to_idx) trainKeys = list(data.keys()) executor = ThreadPoolExecutor(max_workers=1)#params.num_workers) while True: if class_aware_sampling: #idx = np.random.permutation(N) # class-aware supersampling idx = [] num_labels = len(label_to_idx) assert num_labels == params.num_labels for _ in range(N): random_label = np.random.randint(num_labels) if len(running_label_to_idx[random_label]) == 0: running_label_to_idx[random_label] = copy.copy(label_to_idx[random_label]) random.shuffle(running_label_to_idx[random_label]) idx.append(running_label_to_idx[random_label].pop()) else: idx = np.random.permutation(N) batchInds = get_batch_inds(params.batch_size, idx, N) for inds in batchInds: batchKeys = [trainKeys[ind] for ind in inds] codesMetadata, labels = load_lstm_batch(params, data, batchKeys, metadataStats, codesStats, executor, temporal_dropout) yield(codesMetadata,labels)
def img_metadata_generator(params, data, metadataStats, class_aware_sampling = True, augmentation = True): """ Custom generator that yields images or (image,metadata) batches and their category labels (categorical format). :param params: global parameters, used to find location of the dataset and json file :param data: list of objects containing the category labels and paths to images and metadata features :param metadataStats: metadata stats used to normalize metadata features :yield (imgdata,labels) or (imgdata,metadata,labels): image data, metadata (if params set to use), and labels (categorical form) """ N = len(data) if class_aware_sampling: if params.views == 0: data_labels = [datum['category'] for datum in data] else: data_labels = [datum[0]['category'] for datum in data] label_to_idx = defaultdict(list) for i, label in enumerate(data_labels): label_to_idx[label].append(i) running_label_to_idx = copy.deepcopy(label_to_idx) executor = ThreadPoolExecutor(max_workers=params.num_workers) while True: if class_aware_sampling: # class-aware supersampling idx = [] num_labels = len(label_to_idx) assert num_labels == params.num_labels for _ in range(N): random_label = np.random.randint(num_labels) if len(running_label_to_idx[random_label]) == 0: running_label_to_idx[random_label] = copy.copy(label_to_idx[random_label]) random.shuffle(running_label_to_idx[random_label]) idx.append(running_label_to_idx[random_label].pop()) else: idx = np.random.permutation(N) batchInds = get_batch_inds(params.batch_size, idx, N) for inds in batchInds: batchData = [data[ind] for ind in inds] imgdata,metadata,labels = load_cnn_batch(params, batchData, metadataStats, executor, augmentation) inputs = imgdata if params.views != 0: assert len(imgdata) == params.views assert len(metadata) == params.views assert len(labels) == params.views # all labels should be equal label = labels[0] for _label in labels: assert np.argmax(_label) == np.argmax(label) labels = label metadata = np.mean(metadata, axis=0) if params.use_metadata: if not isinstance(inputs, (list, tuple)): inputs = [inputs] inputs.append(metadata) yield(inputs,labels)