def test_lagged_iterator_1d(self): n = 57 chunksize = 10 lag = 1 data = [np.arange(n), np.arange(50), np.arange(30)] input_lens = [x.shape[0] for x in data] reader = DataInMemory(data) reader.chunksize = chunksize self.assertEqual(reader.n_frames_total(), sum(input_lens)) # store results by traj chunked_trajs = [[] for _ in range(len(data))] chunked_lagged_trajs = [[] for _ in range(len(data))] # iterate over data for itraj, X, Y in reader.iterator(lag=lag): chunked_trajs[itraj].append(X) chunked_lagged_trajs[itraj].append(Y) trajs = [np.vstack(ichunks) for ichunks in chunked_trajs] lagged_trajs = [np.vstack(ichunks) for ichunks in chunked_lagged_trajs] # unlagged data for traj, input_traj in zip(trajs, data): np.testing.assert_equal(traj.reshape(input_traj.shape), input_traj) # lagged data lagged_0 = [d[lag:] for d in data] for traj, input_traj in zip(lagged_trajs, lagged_0): np.testing.assert_equal(traj.reshape(input_traj.shape), input_traj)
def test_lagged_iterator_1d(self): n = 30 chunksize = 10 lag = 9 stride = 2 data = [np.arange(n), np.arange(50), np.arange(33)] input_lens = [x.shape[0] for x in data] reader = DataInMemory(data, chunksize=chunksize) it = reader.iterator(chunk=chunksize, stride=stride, lag=lag) # lag < chunksize, so we expect a LaggedIter from pyemma.coordinates.data._base.iterable import _LaggedIterator self.assertIsInstance(it, _LaggedIterator) assert reader.chunksize == chunksize self.assertEqual(reader.n_frames_total(), sum(input_lens)) # store results by traj chunked_trajs = [[] for _ in range(len(data))] chunked_lagged_trajs = [[] for _ in range(len(data))] # iterate over data for itraj, X, Y in reader.iterator(lag=lag, stride=stride): chunked_trajs[itraj].append(X) chunked_lagged_trajs[itraj].append(Y) trajs = [np.vstack(ichunks) for ichunks in chunked_trajs] lagged_trajs = [np.vstack(ichunks) for ichunks in chunked_lagged_trajs] # unlagged data for idx, (traj, input_traj) in enumerate(zip(trajs, data)): # do not consider chunks that have no lagged counterpart input_shape = input_traj.shape np.testing.assert_equal( traj.T.squeeze(), input_traj[::stride][:len(lagged_trajs[idx])].squeeze(), err_msg="failed for traj=%s" % idx) # lagged data for idx, (traj, input_traj) in enumerate(zip(lagged_trajs, data)): np.testing.assert_equal(traj.T.squeeze(), input_traj[lag::stride].squeeze(), err_msg="failed for traj=%s" % idx)
def test_lagged_stridden_access(self): data = np.random.random((1000, 2)) reader = DataInMemory(data) strides = [2, 3, 5, 7, 15] lags = [1, 3, 7, 10, 30] for stride in strides: for lag in lags: chunks = [] for _, _, Y in reader.iterator(stride, lag): chunks.append(Y) chunks = np.vstack(chunks) np.testing.assert_equal(chunks, data[lag::stride])
def test_lagged_iterator_2d(self): chunksize = 10 lag = 1 data = [ np.arange(300).reshape((100, 3)), np.arange(29 * 3).reshape((29, 3)), np.arange(150).reshape(50, 3) ] input_lens = [x.shape[0] for x in data] # print data[0].shape reader = DataInMemory(data) reader.chunksize = chunksize self.assertEqual(reader.n_frames_total(), sum(input_lens)) # store results by traj chunks = [[] for _ in range(len(data))] lagged_chunks = [[] for _ in range(len(data))] # iterate over data for itraj, X, Y in reader.iterator(lag=lag): chunks[itraj].append(X) lagged_chunks[itraj].append(Y) trajs = [np.vstack(ichunks) for ichunks in chunks] lagged_trajs = [np.vstack(ichunks) for ichunks in lagged_chunks] # unlagged data for traj, input_traj in zip(trajs, data): # do not consider chunks that have no lagged counterpart input_shape = input_traj.shape np.testing.assert_equal(traj.reshape((input_shape[0] - lag, 3)), input_traj[:len(input_traj) - lag]) # lagged data lagged_0 = [d[lag:] for d in data] for traj, input_traj in zip(lagged_trajs, lagged_0): np.testing.assert_equal(traj.reshape(input_traj.shape), input_traj)
def test_time_lagged_chunked_access(self): n = 100 data = [np.random.random((n, 3)), np.zeros((29, 3)), np.random.random((n - 50, 3))] reader = DataInMemory(data) self.assertEqual(reader.n_frames_total(), n + n - 50 + 29) # iterate over data it = reader.iterator(lag=30, return_trajindex=True) for itraj, X, Y in it: if itraj == 0: # self.assertEqual(X.shape, (100, 3)) <-- changed behavior: return only chunks of same size self.assertEqual(X.shape, (70, 3)) self.assertEqual(Y.shape, (70, 3)) elif itraj == 1: # the time lagged chunk can not be built due to lag time self.assertEqual(X.shape, (0, 3)) self.assertEqual(Y.shape, (0, 3)) elif itraj == 2: self.assertEqual(X.shape, (20, 3)) self.assertEqual(Y.shape, (20, 3))
def test_lagged_iterator_2d(self): n = 57 chunksize = 10 lag = 1 # data = [np.random.random((n, 3)), # np.zeros((29, 3)), # np.random.random((n - 50, 3))] data = [np.arange(300).reshape((100, 3)), np.arange(29 * 3).reshape((29, 3)), np.arange(150).reshape(50, 3)] input_lens = [x.shape[0] for x in data] # print data[0].shape reader = DataInMemory(data) reader.chunksize = chunksize self.assertEqual(reader.n_frames_total(), sum(input_lens)) # store results by traj chunks = [[] for _ in xrange(len(data))] lagged_chunks = [[] for _ in xrange(len(data))] # iterate over data for itraj, X, Y in reader.iterator(lag=lag): chunks[itraj].append(X) lagged_chunks[itraj].append(Y) trajs = [np.vstack(ichunks) for ichunks in chunks] lagged_trajs = [np.vstack(ichunks) for ichunks in lagged_chunks] # unlagged data for traj, input_traj in zip(trajs, data): np.testing.assert_equal(traj.reshape(input_traj.shape), input_traj) # lagged data lagged_0 = [d[lag:] for d in data] for traj, input_traj in zip(lagged_trajs, lagged_0): np.testing.assert_equal(traj.reshape(input_traj.shape), input_traj)
def test_cols(self): reader = DataInMemory(self.d) cols = (2, 0) for x in reader.iterator(chunk=0, return_trajindex=False, cols=cols): np.testing.assert_equal(x, self.d[:, cols])