def codes_metadata_generator(params, data, metadataStats, codesStats):
"""
Custom generator that yields a vector containign the 4096-d CNN codes output by ResNet50 and metadata features (if params set to use).
:param params: global parameters, used to find location of the dataset and json file
:param data: list of objects containing the category labels and paths to CNN codes and images
:param metadataStats: metadata stats used to normalize metadata features
:yield (codesMetadata,labels): 4096-d CNN codes + metadata features (if set), and labels (categorical form)
"""
N = len(data)
idx = np.random.permutation(N)
batchInds = get_batch_inds(params.batch_size_lstm, idx, N)
trainKeys = list(data.keys())
executor = ProcessPoolExecutor(max_workers=params.num_workers)
while True:
for inds in batchInds:
batchKeys = [trainKeys[ind] for ind in inds]
codesMetadata, labels = load_lstm_batch(params, data, batchKeys,
metadataStats, codesStats,
executor)
yield (codesMetadata, labels)
def img_metadata_generator(params, data, metadataStats):
"""
Custom generator that yields images or (image,metadata) batches and their
category labels (categorical format).
:param params: global parameters, used to find location of the dataset and json file
:param data: list of objects containing the category labels and paths to images and metadata features
:param metadataStats: metadata stats used to normalize metadata features
:yield (imgdata,labels) or (imgdata,metadata,labels): image data, metadata (if params set to use), and labels (categorical form)
"""
N = len(data)
idx = np.random.permutation(N)
batchInds = get_batch_inds(params.batch_size_cnn, idx, N)
executor = ProcessPoolExecutor(max_workers=params.num_workers)
while True:
for inds in batchInds:
batchData = [data[ind] for ind in inds]
imgdata, metadata, labels = load_cnn_batch(params, batchData,
metadataStats, executor)
if params.use_metadata:
yield ([imgdata, metadata], labels)
else:
yield (imgdata, labels)
def img_metadata_generator(params, data, metadataStats):
"""
Custom generator that yields images or (image,metadata) batches and their
category labels (categorical format).
:param params: global parameters, used to find location of the dataset and json file
:param data: list of objects containing the category labels and paths to images and metadata features
:param metadataStats: metadata stats used to normalize metadata features
:yield (imgdata,labels) or (imgdata,metadata,labels): image data, metadata (if params set to use), and labels (categorical form)
"""
N = len(data)
idx = np.random.permutation(N)
batchInds = get_batch_inds(params.batch_size_cnn, idx, N)
executor = ProcessPoolExecutor(max_workers=params.num_workers)
while True:
for inds in batchInds:
batchData = [data[ind] for ind in inds]
imgdata, metadata, labels = load_cnn_batch(params, batchData,
metadataStats, executor)
if (params.generator == 'flip'):
datagen = ImageDataGenerator(horizontal_flip=True,
vertical_flip=True)
elif (params.generator == 'zoom'):
datagen = ImageDataGenerator(zoom_range=[0.9, 1.0],
horizontal_flip=True,
vertical_flip=True)
elif (params.generator == 'shift'):
datagen = ImageDataGenerator(width_shift_range=0.2,
height_shift_range=0.2,
horizontal_flip=True,
vertical_flip=True)
#datagen.fit(imgdata)
#batches = datagen.flow(imgdata, labels, batch_size=params.batch_size_cnn, shuffle=False, save_to_dir='output', save_prefix='aug', save_format='jpg')
batches = datagen.flow(imgdata,
labels,
batch_size=params.batch_size_cnn,
shuffle=False)
idx0 = 0
for batch in batches:
#print (idx)
idx1 = idx0 + batch[0].shape[0]
if params.use_metadata:
#yield ([imgdata, metadata], labels)
yield ([batch[0], metadata[idx0:idx1]], batch[1])
else:
#yield (imgdata, labels)
yield (batch[0], batch[1])
idx0 = idx1
if idx1 >= imgdata.shape[0]:
break
def codes_metadata_generator(params, data, metadataStats, codesStats, class_aware_sampling = True, temporal_dropout = True):
"""
Custom generator that yields a vector containign the 4096-d CNN codes output by ResNet50 and metadata features (if params set to use).
:param params: global parameters, used to find location of the dataset and json file
:param data: list of objects containing the category labels and paths to CNN codes and images
:param metadataStats: metadata stats used to normalize metadata features
:yield (codesMetadata,labels): 4096-d CNN codes + metadata features (if set), and labels (categorical form)
"""
N = len(data)
if class_aware_sampling:
data_labels = [datum['category'] for datum in data.values()]
label_to_idx = defaultdict(list)
for i, label in enumerate(data_labels):
label_to_idx[label].append(i)
running_label_to_idx = copy.deepcopy(label_to_idx)
trainKeys = list(data.keys())
executor = ThreadPoolExecutor(max_workers=1)#params.num_workers)
while True:
if class_aware_sampling:
#idx = np.random.permutation(N)
# class-aware supersampling
idx = []
num_labels = len(label_to_idx)
assert num_labels == params.num_labels
for _ in range(N):
random_label = np.random.randint(num_labels)
if len(running_label_to_idx[random_label]) == 0:
running_label_to_idx[random_label] = copy.copy(label_to_idx[random_label])
random.shuffle(running_label_to_idx[random_label])
idx.append(running_label_to_idx[random_label].pop())
else:
idx = np.random.permutation(N)
batchInds = get_batch_inds(params.batch_size, idx, N)
for inds in batchInds:
batchKeys = [trainKeys[ind] for ind in inds]
codesMetadata, labels = load_lstm_batch(params, data, batchKeys, metadataStats, codesStats, executor, temporal_dropout)
yield(codesMetadata,labels)
def img_metadata_generator(params, data, metadataStats, class_aware_sampling = True, augmentation = True):
"""
Custom generator that yields images or (image,metadata) batches and their
category labels (categorical format).
:param params: global parameters, used to find location of the dataset and json file
:param data: list of objects containing the category labels and paths to images and metadata features
:param metadataStats: metadata stats used to normalize metadata features
:yield (imgdata,labels) or (imgdata,metadata,labels): image data, metadata (if params set to use), and labels (categorical form)
"""
N = len(data)
if class_aware_sampling:
if params.views == 0:
data_labels = [datum['category'] for datum in data]
else:
data_labels = [datum[0]['category'] for datum in data]
label_to_idx = defaultdict(list)
for i, label in enumerate(data_labels):
label_to_idx[label].append(i)
running_label_to_idx = copy.deepcopy(label_to_idx)
executor = ThreadPoolExecutor(max_workers=params.num_workers)
while True:
if class_aware_sampling:
# class-aware supersampling
idx = []
num_labels = len(label_to_idx)
assert num_labels == params.num_labels
for _ in range(N):
random_label = np.random.randint(num_labels)
if len(running_label_to_idx[random_label]) == 0:
running_label_to_idx[random_label] = copy.copy(label_to_idx[random_label])
random.shuffle(running_label_to_idx[random_label])
idx.append(running_label_to_idx[random_label].pop())
else:
idx = np.random.permutation(N)
batchInds = get_batch_inds(params.batch_size, idx, N)
for inds in batchInds:
batchData = [data[ind] for ind in inds]
imgdata,metadata,labels = load_cnn_batch(params, batchData, metadataStats, executor, augmentation)
inputs = imgdata
if params.views != 0:
assert len(imgdata) == params.views
assert len(metadata) == params.views
assert len(labels) == params.views
# all labels should be equal
label = labels[0]
for _label in labels:
assert np.argmax(_label) == np.argmax(label)
labels = label
metadata = np.mean(metadata, axis=0)
if params.use_metadata:
if not isinstance(inputs, (list, tuple)):
inputs = [inputs]
inputs.append(metadata)
yield(inputs,labels)