def test_skip(self): r = DataInMemory(self.d) lagged_it = r.iterator(lag=5) assert lagged_it._it.skip == 0 assert lagged_it._it_lagged.skip == 5 it = r.iterator() for itraj, X in it: if itraj == 0: it.skip = 5 if itraj == 1: assert it.skip == 5
def test_current_trajindex(self): r = DataInMemory(self.d) expected_itraj = 0 for itraj, X in r.iterator(chunk=0): assert itraj == expected_itraj expected_itraj += 1 expected_itraj = -1 it = r.iterator(chunk=16) for itraj, X in it: if it.pos == 0: expected_itraj += 1 assert itraj == expected_itraj == it.current_trajindex
def test_last_chunk(self): r = DataInMemory(self.d) it = r.iterator(chunk=0) for itraj, X in it: assert it.last_chunk_in_traj if itraj == 2: assert it.last_chunk
def test_chunksize_max_memory(self): from pyemma.util.contexts import settings data = np.random.random((10000, 10)) max_size = 1024 with settings(default_chunksize=str(max_size)): r = DataInMemory(data) for itraj, x in r.iterator(): self.assertLessEqual(x.nbytes, max_size)
def test_n_chunks(self): r = DataInMemory(self.d) it0 = r.iterator(chunk=0) assert it0.n_chunks == 3 # 3 trajs it1 = r.iterator(chunk=50) assert it1.n_chunks == 3 * 2 # 2 chunks per trajectory it2 = r.iterator(chunk=30) # 3 full chunks and 1 small chunk per trajectory assert it2.n_chunks == 3 * 4 it3 = r.iterator(chunk=30) it3.skip = 10 assert it3.n_chunks == 3 * 3 # 3 full chunks per traj it4 = r.iterator(chunk=30) it4.skip = 5 # 3 full chunks and 1 chunk of 5 frames per trajectory assert it4.n_chunks == 3 * 4 # test for lagged iterator for stride in range(1, 5): for lag in range(0, 18): it = r.iterator(lag=lag, chunk=30, stride=stride, return_trajindex=False) chunks = sum(1 for _ in it) np.testing.assert_equal( it.n_chunks, chunks, err_msg= "Expected number of chunks did not agree with what the iterator " "returned for stride=%s, lag=%s" % (stride, lag)) assert chunks == it.n_chunks dd = [ np.random.random((100, 3)), np.random.random((120, 3)), np.random.random((120, 3)) ] rr = DataInMemory(dd) # test for lagged iterator for stride in range(1, 5): for lag in [x for x in range(0, 18)] + [50, 100]: it = rr.iterator(lag=lag, chunk=30, stride=stride, return_trajindex=False) chunks = sum(1 for _ in it) np.testing.assert_equal( it.n_chunks, chunks, err_msg= "Expected number of chunks did not agree with what the iterator " "returned for stride=%s, lag=%s" % (stride, lag)) assert chunks == it.n_chunks
def test_invalid_data_in_input_inf(self): self.d[1][-1] = np.inf r = DataInMemory(self.d, chunksize=5) it = r.iterator() from pyemma.coordinates.data._base.datasource import InvalidDataInStreamException with settings(coordinates_check_output=True): with self.assertRaises(InvalidDataInStreamException) as cm: for itraj, X in it: pass
def test_stride(self): r = DataInMemory(self.d) stride = np.arange(1, 17) i = 0 it = r.iterator(stride=stride[i], chunk=1) for _ in it: i += 1 i %= len(stride) it.stride = stride[i] assert it.stride == stride[i]
def test_pos(self): r = DataInMemory(self.d) r.chunksize = 17 it = r.iterator() t = 0 for itraj, X in it: assert t == it.pos t += len(X) if it.last_chunk_in_traj: t = 0
def test_return_trajindex(self): r = DataInMemory(self.d) it = r.iterator(chunk=0) it.return_traj_index = True assert it.return_traj_index is True for tup in it: assert len(tup) == 2 it.reset() it.return_traj_index = False assert it.return_traj_index is False itraj = 0 for tup in it: np.testing.assert_equal(tup, self.d[itraj]) itraj += 1 for tup in r.iterator(return_trajindex=True): assert len(tup) == 2 itraj = 0 for tup in r.iterator(return_trajindex=False): np.testing.assert_equal(tup, self.d[itraj]) itraj += 1
def test_iterator_context(self): dim = DataInMemory(np.array([1])) ctx = dim.iterator(stride=1).state assert ctx.stride == 1 assert ctx.uniform_stride assert ctx.is_stride_sorted() assert ctx.traj_keys is None ctx = dim.iterator(stride=np.asarray([[0, 0], [0, 1], [0, 2]])).state assert not ctx.uniform_stride assert ctx.is_stride_sorted() np.testing.assert_array_equal(ctx.traj_keys, np.array([0])) # require sorted random access dim._needs_sorted_random_access_stride = True # sorted within trajectory, not sorted by trajectory key with self.assertRaises(ValueError): dim.iterator(stride=np.asarray([[1, 1], [1, 2], [1, 3], [0, 0], [0, 1], [0, 2]])) # sorted by trajectory key, not within trajectory with self.assertRaises(ValueError): dim.iterator(stride=np.asarray([[0, 0], [0, 1], [0, 2], [1, 1], [1, 5], [1, 3]])) np.testing.assert_array_equal(ctx.ra_indices_for_traj(0), np.array([0, 1, 2]))
def test_n_chunks(self): r = DataInMemory(self.d) it0 = r.iterator(chunk=0) assert it0._n_chunks == 3 # 3 trajs it1 = r.iterator(chunk=50) assert it1._n_chunks == 3 * 2 # 2 chunks per trajectory it2 = r.iterator(chunk=30) # 3 full chunks and 1 small chunk per trajectory assert it2._n_chunks == 3 * 4 it3 = r.iterator(chunk=30) it3.skip = 10 assert it3._n_chunks == 3 * 3 # 3 full chunks per traj it4 = r.iterator(chunk=30) it4.skip = 5 # 3 full chunks and 1 chunk of 5 frames per trajectory assert it4._n_chunks == 3 * 4 # test for lagged iterator for stride in range(1, 5): for lag in range(0, 18): it = r.iterator(lag=lag, chunk=30, stride=stride, return_trajindex=False) chunks = 0 for _ in it: chunks += 1 assert chunks == it._n_chunks
def test_chunksize(self): r = DataInMemory(self.d) cs = np.arange(1, 17) i = 0 it = r.iterator(chunk=cs[i]) for itraj, X in it: if not it.last_chunk_in_traj: assert len(X) == it.chunksize else: assert len(X) <= it.chunksize i += 1 i %= len(cs) it.chunksize = cs[i] assert it.chunksize == cs[i]
def test_n_chunks_ra(self): """ """ r = DataInMemory(self.d) def gen_sorted_stride(n): frames = np.random.randint(0, 99, size=n) trajs = np.random.randint(0, 3, size=n) stride = np.sort(np.stack((trajs, frames)).T, axis=1) # sort by file and frame index sort_inds = np.lexsort((stride[:, 1], stride[:, 0])) return stride[sort_inds] strides = [gen_sorted_stride(np.random.randint(1, 99)) for _ in range(10)] lengths = [len(x) for x in strides] for chunk in range(0, 100):#max(lengths)): for stride in strides: it = r.iterator(chunk=chunk, stride=stride) self._count_chunks(it)