def test_serialize_incremental_chunk_size_pandas_to_recarray(input_df_descr): if not is_test_data_serializable(input_df_descr): return df = _mixed_test_data()[input_df_descr][0] expectation = _mixed_test_data()[input_df_descr][1] if not isinstance(expectation, tuple) and issubclass( expectation, Exception): for div in (1, 4, 8): chunk_size = div * 8 * 1024**2 with pytest.raises(expectation): incr_ser = IncrementalPandasToRecArraySerializer( df_serializer, df, chunk_size=chunk_size) [chunk for chunk, _, _, _ in incr_ser.generator_bytes()] return for div in (1, 4, 8): chunk_size = div * 8 * 1024**2 if input_df_descr is not None and len(expectation) > 0: row_size = int(expectation[0].dtype.itemsize) chunk_size = NON_HOMOGENEOUS_DTYPE_PATCH_SIZE_ROWS * row_size / div incr_ser = IncrementalPandasToRecArraySerializer(df_serializer, df, chunk_size=chunk_size) chunk_bytes = [chunk for chunk, _, _, _ in incr_ser.generator_bytes()] matching = expectation[0].tostring() == b''.join(chunk_bytes) assert matching assert expectation[1] == incr_ser.dtype
def test_shape(input_df_descr): if not is_test_data_serializable(input_df_descr): return df = _mixed_test_data()[input_df_descr][0] expectation = _mixed_test_data()[input_df_descr][1] incr_ser = IncrementalPandasToRecArraySerializer(df_serializer, df, chunk_size=_CHUNK_SIZE) if not isinstance(expectation, tuple) and issubclass( expectation, Exception): with pytest.raises(expectation): [chunk for chunk, _, _, _ in incr_ser.shape] else: assert incr_ser.shape == expectation[0].shape
def test_generator_bytes_range(from_idx, to_idx): # Tests also negative indexing df = _mixed_test_data()['large'][0] expectation = _mixed_test_data()['large'][1] incr_ser = IncrementalPandasToRecArraySerializer(df_serializer, df, chunk_size=_CHUNK_SIZE) chunk_bytes = [ chunk_b for chunk_b, _, _, _ in incr_ser.generator_bytes(from_idx=from_idx, to_idx=to_idx) ] matching = expectation[0][from_idx:to_idx].tostring() == b''.join( chunk_bytes) assert matching assert expectation[1] == incr_ser.dtype
def test_serialize_pandas_to_recarray(input_df_descr): if not is_test_data_serializable(input_df_descr): return df = _mixed_test_data()[input_df_descr][0] expectation = _mixed_test_data()[input_df_descr][1] incr_ser = IncrementalPandasToRecArraySerializer(df_serializer, df, chunk_size=_CHUNK_SIZE) if not isinstance(expectation, tuple) and issubclass( expectation, Exception): with pytest.raises(expectation): [chunk for chunk, _, _, _ in incr_ser.generator_bytes()] else: incr_ser_data, incr_ser_dtype = incr_ser.serialize() matching = expectation[0].tostring() == incr_ser_data.tostring() assert matching assert expectation[1] == incr_ser_dtype
def test_incremental_bad_init(): with pytest.raises(ArcticSerializationException): IncrementalPandasToRecArraySerializer(df_serializer, 'hello world', chunk_size=_CHUNK_SIZE) with pytest.raises(ArcticSerializationException): IncrementalPandasToRecArraySerializer(df_serializer, 1234, chunk_size=_CHUNK_SIZE) with pytest.raises(ArcticSerializationException): IncrementalPandasToRecArraySerializer(df_serializer, _mixed_test_data()['small'][0], chunk_size=0) with pytest.raises(ArcticSerializationException): IncrementalPandasToRecArraySerializer(df_serializer, _mixed_test_data()['small'][0], chunk_size=-1) with pytest.raises(ArcticSerializationException): IncrementalPandasToRecArraySerializer(df_serializer, _mixed_test_data()['small'][0], chunk_size=_CHUNK_SIZE, string_max_len=-1)
chunk_size=_CHUNK_SIZE, string_max_len=-1) def test_none_df(): with pytest.raises(ArcticSerializationException): incr_ser = IncrementalPandasToRecArraySerializer( df_serializer, None, chunk_size=_CHUNK_SIZE) incr_ser.serialize() with pytest.raises(ArcticSerializationException): incr_ser = IncrementalPandasToRecArraySerializer( df_serializer, None, chunk_size=_CHUNK_SIZE) incr_ser.generator_bytes() @pytest.mark.parametrize("input_df_descr", _mixed_test_data().keys()) def test_serialize_pandas_to_recarray(input_df_descr): if not is_test_data_serializable(input_df_descr): return df = _mixed_test_data()[input_df_descr][0] expectation = _mixed_test_data()[input_df_descr][1] incr_ser = IncrementalPandasToRecArraySerializer(df_serializer, df, chunk_size=_CHUNK_SIZE) if not isinstance(expectation, tuple) and issubclass( expectation, Exception): with pytest.raises(expectation): [chunk for chunk, _, _, _ in incr_ser.generator_bytes()] else: