def test_read_csv_data_split_limited_object(): total_memory_in_bytes = psutil.virtual_memory().total two_mb_in_bytes = _convert_megabytes_to_bytes(2) fraction_of_memory_to_use = two_mb_in_bytes / total_memory_in_bytes X, y = read_csv_data( LARGE_DATA_4MB, target_column_index=0, fit_memory_percent=fraction_of_memory_to_use * 100, output_dtype="O" ) arrays_memory = _get_size_total(X) + _get_size_total(y) assert _convert_megabytes_to_bytes(1.9) < arrays_memory <= two_mb_in_bytes assert X.dtype.kind == "O" assert y.dtype.kind == "O"
def test_read_csv_data_samples_object(): """Test for sample case where the entire dataset doesn't fit into the available memory""" total_memory_in_bytes = psutil.virtual_memory().total two_mb_in_bytes = _convert_megabytes_to_bytes(2) fraction_of_memory_to_use = two_mb_in_bytes / total_memory_in_bytes sample_data = read_csv_data( source=LARGE_DATA_4MB, fit_memory_percent=fraction_of_memory_to_use * 100, output_dtype="object" ) array_memory = _get_size_total(sample_data) assert _convert_megabytes_to_bytes(1.9) < array_memory <= two_mb_in_bytes assert sample_data.dtype.kind == "O"