def __call__(self, batch: List[List[str]], *args,
**kwargs) -> Union[List[np.ndarray], np.ndarray]:
"""
Embed sentences from a batch.
Args:
batch: A list of tokenized text samples.
Returns:
A batch of ELMo embeddings.
"""
if len(batch) > self.mini_batch_size:
batch_gen = chunk_generator(batch, self.mini_batch_size)
elmo_output_values = []
for mini_batch in batch_gen:
mini_batch_out = self._mini_batch_fit(mini_batch, *args,
**kwargs)
elmo_output_values.extend(mini_batch_out)
else:
elmo_output_values = self._mini_batch_fit(batch, *args, **kwargs)
if self.pad_zero:
elmo_output_values = zero_pad(elmo_output_values)
return elmo_output_values
def _line_generator(self, shard_generator):
for shard in shard_generator:
line_generator = chunk_generator(shard, 1)
for line in line_generator:
line = line[0]
char_ids, reversed_char_ids, token_ids, reversed_token_ids =\
self._line2ids(line)
yield char_ids, reversed_char_ids, token_ids, reversed_token_ids
def gen_batches(self, batch_size: int, data_type: str = 'train', shuffle: Optional[bool] = None)\
-> Iterator[Tuple[str, str]]:
if shuffle is None:
shuffle = self.shuffle
tgt_data = self.data[data_type]
shard_generator = self._shard_generator(tgt_data, shuffle=shuffle)
for shard in shard_generator:
if not (batch_size):
bs = len(shard)
lines_generator = chunk_generator(shard, bs)
for lines in lines_generator:
yield (lines, [None] * len(lines))
def gen_batches(self, batch_size: int, data_type: str = 'train', shuffle: Optional[bool] = None) \
-> Iterator[Tuple[str, str]]:
if shuffle is None:
shuffle = self.shuffle
tgt_data = self.data[data_type]
shard_generator = self._shard_generator(tgt_data, shuffle=shuffle)
for shard in shard_generator:
if not (batch_size):
bs = len(shard)
lines_generator = chunk_generator(shard, bs)
for lines in lines_generator:
yield (lines, [None] * len(lines))
def __call__(self, batch: List[List[str]],
*args, **kwargs) -> Union[List[np.ndarray], np.ndarray]:
"""
Embed sentences from a batch.
Args:
batch: A list of tokenized text samples.
Returns:
A batch of ELMo embeddings.
"""
if len(batch) > self.mini_batch_size:
batch_gen = chunk_generator(batch, self.mini_batch_size)
elmo_output_values = []
for mini_batch in batch_gen:
mini_batch_out = self._mini_batch_fit(mini_batch, *args, **kwargs)
elmo_output_values.extend(mini_batch_out)
else:
elmo_output_values = self._mini_batch_fit(batch, *args, **kwargs)
if self.pad_zero:
elmo_output_values = zero_pad(elmo_output_values)
return elmo_output_values