def test_read_csv_data_split_limited_object():
total_memory_in_bytes = psutil.virtual_memory().total
two_mb_in_bytes = _convert_megabytes_to_bytes(2)
fraction_of_memory_to_use = two_mb_in_bytes / total_memory_in_bytes
X, y = read_csv_data(
LARGE_DATA_4MB, target_column_index=0, fit_memory_percent=fraction_of_memory_to_use * 100, output_dtype="O"
)
arrays_memory = _get_size_total(X) + _get_size_total(y)
assert _convert_megabytes_to_bytes(1.9) < arrays_memory <= two_mb_in_bytes
assert X.dtype.kind == "O"
assert y.dtype.kind == "O"
def test_read_csv_data_samples_object():
"""Test for sample case where the entire dataset doesn't fit into the available memory"""
total_memory_in_bytes = psutil.virtual_memory().total
two_mb_in_bytes = _convert_megabytes_to_bytes(2)
fraction_of_memory_to_use = two_mb_in_bytes / total_memory_in_bytes
sample_data = read_csv_data(
source=LARGE_DATA_4MB, fit_memory_percent=fraction_of_memory_to_use * 100, output_dtype="object"
)
array_memory = _get_size_total(sample_data)
assert _convert_megabytes_to_bytes(1.9) < array_memory <= two_mb_in_bytes
assert sample_data.dtype.kind == "O"