def test_random_split(data): dataframes = list(random_split(data)) assert len(dataframes) > 1 concatenated = pd.concat(dataframes) assert len(concatenated) == len(data), "We did not lose anything" assert np.allclose( concatenated.values, data.values), "We did not corrupt the data"
def test_multiple_chunks(self, data): chunks = random_split(data) chopper = TimeChopper(chunks, 5) result = list(chopper) assert len(result) == MAX_TS concatinated = pd.concat(r[1] for r in result) assert len(data) == len(concatinated), "We did not lose anything"
def test_partially_reversed_data(self, data): results_queue = Queue() results = [] chunks = list(random_split(data)) chunks[5], chunks[6] = chunks[6], chunks[5] pipeline = Aggregator(TimeChopper(DataPoller(source=chunks, poll_period=0.1), cache_size=3), AGGR_CONFIG, False) drain = Drain(pipeline, results_queue) drain.run() assert results_queue.qsize() == MAX_TS
def test_partially_reversed_data(self, data): chunks = list(random_split(data)) chunks[5], chunks[6] = chunks[6], chunks[5] chopper = TimeChopper(chunks, 5) result = list(chopper) assert len(result) == MAX_TS, "DataFrame is splitted into proper number of chunks" concatinated = pd.concat(r[1] for r in result) assert len(data) == len(concatinated), "We did not lose anything" assert np.allclose(concatinated.values, data.values), "We did not corrupt the data"
def test_partially_reversed_data(self, data): results_queue = Queue() chunks = list(random_split(data)) chunks[5], chunks[6] = chunks[6], chunks[5] pipeline = Aggregator( TimeChopper(DataPoller(source=chunks, poll_period=0.1), cache_size=3), AGGR_CONFIG, False) drain = Drain(pipeline, results_queue) drain.run() assert results_queue.qsize() == MAX_TS
def test_partially_reversed_data(self, data): chunks = list(random_split(data)) chunks[5], chunks[6] = chunks[6], chunks[5] chopper = TimeChopper(chunks, 5) result = list(chopper) assert (len(result) == MAX_TS, "DataFrame is splitted into proper number of chunks") concatinated = pd.concat(r[1] for r in result) assert len(data) == len(concatinated), "We did not lose anything" assert np.allclose(concatinated.values, data.values), "We did not corrupt the data"
def test_slow_producer(self, data): results_queue = Queue() chunks = list(random_split(data)) chunks[5], chunks[6] = chunks[6], chunks[5] def producer(): for chunk in chunks: if np.random.random() > 0.5: yield None yield chunk pipeline = Aggregator( TimeChopper(DataPoller(source=producer(), poll_period=0.1), cache_size=3), AGGR_CONFIG, False) drain = Drain(pipeline, results_queue) drain.run() assert results_queue.qsize() == MAX_TS
def test_slow_producer(self, data): results_queue = Queue() results = [] chunks = list(random_split(data)) chunks[5], chunks[6] = chunks[6], chunks[5] def producer(): for chunk in chunks: if np.random.random() > 0.5: yield None yield chunk pipeline = Aggregator( TimeChopper(DataPoller(source=producer(), poll_period=0.1), cache_size=3), AGGR_CONFIG, False ) drain = Drain(pipeline, results_queue) drain.run() assert results_queue.qsize() == MAX_TS