def test_raises_on_non_string(self):
import numpy
with pytest.raises(TypeError):
ppg.MemMappedDataLoadingJob(55, lambda: 5, lambda: 5, numpy.uint32)
with pytest.raises(ValueError):
ppg.MemMappedDataLoadingJob("shu", 5, lambda: 5, numpy.uint32)
with pytest.raises(ValueError):
ppg.MemMappedDataLoadingJob("shu", lambda: 5, 5, numpy.uint32)
def test_invalidation(self, new_pipegraph):
import numpy
o = {}
def calc():
return numpy.array(range(0, 10), dtype=numpy.uint32)
def store(value):
o[0] = value
def cleanup():
del o[0]
dl = ppg.MemMappedDataLoadingJob("out/A", calc, store,
numpy.uint32)
dl.cleanup = cleanup
of = "out/B"
def do_write():
assert isinstance(o[0], numpy.core.memmap)
write(of, ",".join(str(x) for x in o[0]))
append("out/C", "a")
ppg.FileGeneratingJob(of, do_write).depends_on(dl)
ppg.run_pipegraph()
assert read("out/B") == "0,1,2,3,4,5,6,7,8,9"
assert read("out/C") == "a"
# now, no run...
new_pipegraph.new_pipegraph()
dl = ppg.MemMappedDataLoadingJob("out/A", calc, store,
numpy.uint32)
dl.cleanup = cleanup
ppg.FileGeneratingJob(of, do_write).depends_on(dl)
ppg.run_pipegraph()
assert read("out/C") == "a"
new_pipegraph.new_pipegraph()
def calc2():
append("out/D", "a")
return numpy.array(range(0, 12), dtype=numpy.uint32)
dl = ppg.MemMappedDataLoadingJob("out/A", calc2, store,
numpy.uint32)
ppg.FileGeneratingJob(of, do_write).depends_on(dl)
dl.cleanup = cleanup
ppg.run_pipegraph()
assert read("out/D") == "a"
assert read("out/B") == "0,1,2,3,4,5,6,7,8,9,10,11"
assert read("out/C") == "aa"
def test_raises_on_wrong_dtype(self):
import numpy
o = []
def calc():
return numpy.array(range(0, 10), dtype=numpy.float)
def store(value):
o.append(value)
def cleanup():
del o[0]
dl = ppg.MemMappedDataLoadingJob("out/A", calc, store, numpy.uint32)
dl.cleanup = cleanup
of = "out/B"
def do_write():
assert isinstance(o[0], numpy.core.memmap)
write(of, ",".join(str(x) for x in o[0]))
ppg.FileGeneratingJob(of, do_write).depends_on(dl)
def inner():
ppg.run_pipegraph()
assertRaises(ppg.RuntimeError, inner)
assert isinstance(dl.lfg.exception, ppg.JobContractError)
assert dl.failed
def test_simple(self, new_pipegraph):
new_pipegraph.new_pipegraph()
import numpy
o = []
def calc():
return numpy.array(range(0, 10), dtype=numpy.uint32)
def store(value):
o.append(value)
def cleanup():
del o[0]
dl = ppg.MemMappedDataLoadingJob("out/A", calc, store, numpy.uint32)
dl.cleanup = cleanup
of = "out/B"
def do_write():
assert isinstance(o[0], numpy.core.memmap)
write(of, ",".join(str(x) for x in o[0]))
ppg.FileGeneratingJob(of, do_write).depends_on(dl)
dl.depends_on_params(123)
ppg.run_pipegraph()
assert read("out/B") == "0,1,2,3,4,5,6,7,8,9"
def test_ignore_code_changes(self, new_pipegraph):
import numpy
import pathlib
o = []
def calc():
return numpy.array(range(0, 10), dtype=numpy.uint32)
def store(value):
o.append(value)
def cleanup():
o.clear()
dl = ppg.MemMappedDataLoadingJob("out/A", calc, store, numpy.uint32)
dl.cleanup = cleanup
of = "out/B"
def do_write():
assert isinstance(o[0], numpy.core.memmap)
write(of, ",".join(str(x) for x in o[0]))
ppg.FileGeneratingJob(of, do_write).depends_on(dl)
dl.depends_on_params(123)
ppg.run_pipegraph()
assert read("out/B") == "0,1,2,3,4,5,6,7,8,9"
new_pipegraph.new_pipegraph()
def calc2():
return numpy.array(range(0, 10), dtype=numpy.uint32) + 1
dl = ppg.MemMappedDataLoadingJob("out/A", calc2, store, numpy.uint32)
dl.cleanup = cleanup
of = "out/B"
ppg.FileGeneratingJob(of, do_write).depends_on(dl)
dl.depends_on_params(123)
ppg.run_pipegraph()
assert read("out/B") == "1,2,3,4,5,6,7,8,9,10"
new_pipegraph.new_pipegraph()
def calc3():
return numpy.array(range(0, 10), dtype=numpy.uint32) + 1
dl = ppg.MemMappedDataLoadingJob("out/A", calc3, store, numpy.uint32)
dl.ignore_code_changes()
dl.cleanup = cleanup
of = "out/B"
ppg.FileGeneratingJob(of, do_write).depends_on(dl)
dl.depends_on_params(123)
ppg.run_pipegraph()
assert read("out/B") == "1,2,3,4,5,6,7,8,9,10" # jup, no rerun
new_pipegraph.new_pipegraph()
def store2(value):
o.append(value)
o.append(value)
dl = ppg.MemMappedDataLoadingJob("out/A", calc2, store2, numpy.uint32)
dl.ignore_code_changes()
dl.cleanup = cleanup
of = "out/B"
ppg.FileGeneratingJob(of, do_write).depends_on(dl)
dl.depends_on_params(123)
ppg.run_pipegraph()
assert read("out/B") == "1,2,3,4,5,6,7,8,9,10" # jup, no rerun
new_pipegraph.new_pipegraph()
dl = ppg.MemMappedDataLoadingJob("out/A", calc2, store2, numpy.uint32)
dl.cleanup = cleanup
of = "out/B"
ppg.FileGeneratingJob(of, do_write).depends_on(dl)
dl.depends_on_params(123)
write("out/B", "replace me")
ppg.run_pipegraph()
assert read("out/B") == "1,2,3,4,5,6,7,8,9,10" # jup, no rerun
new_pipegraph.new_pipegraph()
dl = ppg.MemMappedDataLoadingJob(
pathlib.Path("out/A"), calc2, store2, numpy.uint32
)
dl.cleanup = cleanup
of = "out/B"
ppg.FileGeneratingJob(of, do_write).depends_on(dl)
dl.depends_on_params(123)
dl.lfg.depends_on_params(
456
) # this should trigger just the load invalidation
write("out/B", "replace me")
ppg.run_pipegraph()
assert read("out/B") == "1,2,3,4,5,6,7,8,9,10" # jup, no rerun
def inner():
ppg.MemMappedDataLoadingJob("out/A", calc, store, numpypy.uint32)
