def merge(self):
if len(self._unique_states_array_ids) > 1:
states_array_lens = map(lambda xx: len(xx._states_array), self)
sector_position = map(lambda xx: slice(*xx), list(tkr.ngram(np.cumsum([0] + states_array_lens), 2)))
join_all_states_arrays = np.concatenate(tuple(map(lambda xx: xx._states_array, self)), axis=0)
u, i = np.unique(join_all_states_arrays, return_inverse=True)
ptrs_transforms = map(lambda xx: i[xx], sector_position)
self._new_states_array = StatesArray(data=u)
self._ptrs_transforms = ptrs_transforms
else:
self._new_states_array = self[0]._states_array
return self
def merge(self):
if len(self._unique_states_array_ids) > 1:
states_array_lens = map(lambda xx: len(xx._states_array), self)
sector_position = map(
lambda xx: slice(*xx),
list(tkr.ngram(np.cumsum([0] + states_array_lens), 2)))
join_all_states_arrays = np.concatenate(tuple(
map(lambda xx: xx._states_array, self)),
axis=0)
u, i = np.unique(join_all_states_arrays, return_inverse=True)
ptrs_transforms = map(lambda xx: i[xx], sector_position)
self._new_states_array = StatesArray(data=u)
self._ptrs_transforms = ptrs_transforms
else:
self._new_states_array = self[0]._states_array
return self
def ngram(self, n):
ngram_results = list(tkr.ngram(self, n))
return type(self)(data=ngram_results, eval_cls=self._eval_cls)
def ngram(self, n):
ngram_results = list(tkr.ngram(self, n))
return type(self)(data=ngram_results, eval_cls=self._eval_cls)