def __call__(self):
if self._num_epochs and self._epoch >= self._num_epochs:
raise errors.OutOfRangeError(
None, None, "Already emitted %s epochs." % self._epoch)
list_dict = {}
list_dict_size = 0
while list_dict_size < self._batch_size:
try:
data_row = next(self._iterator)
except StopIteration:
self._epoch += 1
self._iterator = self._generator_function()
data_row = next(self._iterator)
for index, key in enumerate(self._keys):
if key not in data_row.keys():
raise KeyError(
"key mismatch between dicts emitted by GenFun "
"Expected {} keys; got {}".format(
self._keys, data_row.keys()))
list_dict.setdefault(self._col_placeholders[index],
list()).append(data_row[key])
list_dict_size += 1
if self._pad_value is not None:
feed_dict = {
key: np.asarray(_pad_if_needed(item, self._pad_value))
for key, item in list(list_dict.items())
}
else:
feed_dict = {
key: np.asarray(item)
for key, item in list(list_dict.items())
}
return feed_dict
def __call__(self):
if self._num_epochs and self._epoch >= self._num_epochs:
raise errors.OutOfRangeError(
None, None, "Already emitted %s epochs." % self._epoch)
integer_indexes = [
j % self._max
for j in range(self._trav, self._trav + self._batch_size)
]
if self._epoch_end in integer_indexes:
# after this batch we will have processed self._epoch epochs, possibly
# overshooting a bit to fill out a batch.
self._epoch += 1
if self._epoch == self._num_epochs:
# trim this batch, so as not to overshoot the last epoch.
batch_end_inclusive = integer_indexes.index(self._epoch_end)
integer_indexes = integer_indexes[:(batch_end_inclusive + 1)]
self._trav = (integer_indexes[-1] + 1) % self._max
result = self._dataframe.iloc[integer_indexes]
cols = [result[col].values for col in result.columns]
feed_dict = dict(zip(self._col_placeholders, cols))
feed_dict[self._index_placeholder] = result.index.values
return feed_dict
def __call__(self):
ins = self._i.batch_id_queue.get()
if ins is False:
raise errors.OutOfRangeError(None, None, "Already emitted epochs.")
vids, dense_labels = ins
vid_index = []
for vid_idx, vid in enumerate(vids):
idx = self.vid_dict[vid]
vid_index.append(idx)
vid_index = np.array(vid_index)
vid_index_sortidx = np.argsort(vid_index)
try:
batch_data = self._i.mean_data[vid_index[vid_index_sortidx], :]
except:
print(vid_index[vid_index_sortidx])
print(vids)
print("\n")
batch_data = batch_data[np.argsort(vid_index_sortidx), :]
vals = [np.array(vids), dense_labels, batch_data]
feed_dict = {}
for pl, val in zip(self.placeholders, vals):
feed_dict[pl.name] = val
return feed_dict
def _get_integer_indices_for_next_batch(batch_indices_start, batch_size,
epoch_end, array_length, current_epoch,
total_epochs):
"""Returns the integer indices for next batch.
If total epochs is not None and current epoch is the final epoch, the end
index of the next batch should not exceed the `epoch_end` (i.e., the final
batch might not have size `batch_size` to avoid overshooting the last epoch).
Args:
batch_indices_start: Integer, the index to start next batch.
batch_size: Integer, size of batches to return.
epoch_end: Integer, the end index of the epoch. The epoch could start from a
random position, so `epoch_end` provides the end index for that.
array_length: Integer, the length of the array.
current_epoch: Integer, the epoch number has been emitted.
total_epochs: Integer or `None`, the total number of epochs to emit. If
`None` will run forever.
Returns:
A tuple of a list with integer indices for next batch and `current_epoch`
value after the next batch.
Raises:
OutOfRangeError if `current_epoch` is not less than `total_epochs`.
"""
if total_epochs is not None and current_epoch >= total_epochs:
raise errors.OutOfRangeError(
None, None, "Already emitted %s epochs." % current_epoch)
batch_indices_end = batch_indices_start + batch_size
batch_indices = [
j % array_length for j in range(batch_indices_start, batch_indices_end)
]
epoch_end_indices = [
i for i, x in enumerate(batch_indices) if x == epoch_end
]
current_epoch += len(epoch_end_indices)
if total_epochs is None or current_epoch < total_epochs:
return (batch_indices, current_epoch)
# Now we might have emitted more data for expected epochs. Need to trim.
final_epoch_end_inclusive = epoch_end_indices[-(current_epoch -
total_epochs + 1)]
batch_indices = batch_indices[:final_epoch_end_inclusive + 1]
return (batch_indices, total_epochs)
def __call__(self):
if self._num_epochs and self._epoch >= self._num_epochs:
raise errors.OutOfRangeError(
None, None, "Already emitted %s epochs." % self._epoch)
integer_indexes = [
j % self._max
for j in range(self._trav, self._trav + self._batch_size)
]
if self._epoch_end in integer_indexes:
# after this batch we will have processed self._epoch epochs, possibly
# overshooting a bit to fill out a batch.
self._epoch += 1
self._trav = (integer_indexes[-1] + 1) % self._max
return {
self._placeholders[0]: integer_indexes,
self._placeholders[1]: self._array[integer_indexes]
}
def __call__(self):
ins = self._i.batch_id_queue.get()
if ins is False:
raise errors.OutOfRangeError(None, None,
"Already emitted epochs.")
vids, dense_labels = ins
batch_data = np.zeros((len(vids), 256*256), dtype=np.float32)
for vid_idx, vid in enumerate(vids):
fidx, inside_f_idx = self.vid_dict[vid]
d = self._i.mean_data_list[fidx][inside_f_idx, :]
d = np.reshape(d, [256, 256])
d = np.sign(d) * np.sqrt(np.absolute(d))
d = normalize(d)
batch_data[vid_idx] = np.reshape(d, [-1])
vals = [np.array(vids), dense_labels, batch_data]
feed_dict = {}
for pl, val in zip(self.placeholders, vals):
feed_dict[pl.name] = val
return feed_dict
def __call__(self):
if self._num_epochs and self._epoch >= self._num_epochs:
raise errors.OutOfRangeError(
None, None, "Already emitted %s epochs." % self._epoch)
integer_indexes = [
j % self._max
for j in range(self._trav, self._trav + self._batch_size)
]
if self._epoch_end in integer_indexes:
# after this batch we will have processed self._epoch epochs, possibly
# overshooting a bit to fill out a batch.
self._epoch += 1
self._trav = (integer_indexes[-1] + 1) % self._max
feed_dict = {self._index_placeholder: integer_indexes}
cols = [
column[integer_indexes]
for column in self._ordered_dict_of_arrays.values()
]
feed_dict.update(dict(zip(self._col_placeholders, cols)))
return feed_dict
def __call__(self):
if self._num_epochs is not None and self._epoch >= self._num_epochs:
raise errors.OutOfRangeError(
None, None, "Already emitted %s epochs." % self._epoch)
indices_end = self._trav + self._batch_size
# _num_epochs will not be <= 0; otherwise, the OutOfRangeError has been
# raised already.
if self._num_epochs is not None and self._epoch == self._num_epochs - 1:
# If num_epochs is set and the feed_fn is on the final epoch, the end
# index of the next batch should not exceed the epoch_end.
if self._trav <= self._epoch_end:
epoch_end = self._epoch_end
else:
epoch_end = self._max + self._epoch_end
indices_end = min(epoch_end + 1, indices_end)
# The integer indices for next batch.
integer_indexes = [
j % self._max for j in range(self._trav, indices_end)
]
if self._epoch_end in integer_indexes:
# after this batch we will have processed self._epoch epochs, possibly
# overshooting a bit to fill out a batch.
self._epoch += 1
self._trav = (integer_indexes[-1] + 1) % self._max
feed_dict = {self._index_placeholder: integer_indexes}
cols = [
column[integer_indexes]
for column in self._ordered_dict_of_arrays.values()
]
feed_dict.update(dict(zip(self._col_placeholders, cols)))
return feed_dict
