def test_memory(mock_get_step_uuid, data_1, test_transfer, plasma_store):
orchest.Config.PIPELINE_DESCRIPTION_PATH = "tests/userdir/pipeline-basic.json"
# Do as if we are uuid-1. Note the trailing underscores. This is to
# make the plasma.ObjectID the required 20 characters.
mock_get_step_uuid.return_value = "uuid-1______________"
test_transfer["method"](data_1, **test_transfer["kwargs"])
# Do as if we are uuid-2
mock_get_step_uuid.return_value = "uuid-2______________"
input_data = transfer.get_inputs()
assert (input_data == data_1).all()
def test_memory_disk_fallback(mock_get_step_uuid, plasma_store):
orchest.Config.PIPELINE_DESCRIPTION_PATH = "tests/userdir/pipeline-basic.json"
# Do as if we are uuid-1
data_1 = generate_data((PLASMA_KILOBYTES + 1) * KILOBYTE)
data_size = pa.serialize(data_1).total_bytes
assert data_size > PLASMA_STORE_CAPACITY
mock_get_step_uuid.return_value = "uuid-1______________"
transfer.output_to_memory(
data_1, disk_fallback=True,
)
# Do as if we are uuid-2
mock_get_step_uuid.return_value = "uuid-2______________"
input_data = transfer.get_inputs()
assert (input_data[0] == data_1).all()
def test_output_no_memory_store(mock_get_step_uuid):
"""Test the order of the inputs of the receiving step.
Note that the order in which the data is output does not determine the
"receive order", it is the order in which it is defined in the
pipeline.json (for the "incoming-connections").
"""
orchest.Config.PIPELINE_DESCRIPTION_PATH = "tests/userdir/pipeline-basic.json"
# Do as if we are uuid-1
data_1 = generate_data(KILOBYTE)
mock_get_step_uuid.return_value = "uuid-1______________"
transfer.output(data_1)
# Do as if we are uuid-2
mock_get_step_uuid.return_value = "uuid-2______________"
input_data = transfer.get_inputs()
assert (input_data[0] == data_1).all()
def test_memory_pickle_fallback_and_disk_fallback(mock_get_step_uuid, plasma_store):
data_1 = [
UnserializableByPyarrowObject(generate_data(KILOBYTE))
for _ in range(PLASMA_KILOBYTES + 1)
]
serialized, _ = transfer.serialize(data_1)
assert serialized.total_bytes > PLASMA_STORE_CAPACITY
orchest.Config.PIPELINE_DESCRIPTION_PATH = "tests/userdir/pipeline-basic.json"
# Do as if we are uuid-1
mock_get_step_uuid.return_value = "uuid-1______________"
transfer.output_to_memory(
data_1, disk_fallback=True,
)
# Do as if we are uuid-2
mock_get_step_uuid.return_value = "uuid-2______________"
input_data = transfer.get_inputs()
assert input_data[0] == data_1
