def test_raises_on_non_string(self): import numpy with pytest.raises(TypeError): ppg.MemMappedDataLoadingJob(55, lambda: 5, lambda: 5, numpy.uint32) with pytest.raises(ValueError): ppg.MemMappedDataLoadingJob("shu", 5, lambda: 5, numpy.uint32) with pytest.raises(ValueError): ppg.MemMappedDataLoadingJob("shu", lambda: 5, 5, numpy.uint32)
def test_invalidation(self, new_pipegraph): import numpy o = {} def calc(): return numpy.array(range(0, 10), dtype=numpy.uint32) def store(value): o[0] = value def cleanup(): del o[0] dl = ppg.MemMappedDataLoadingJob("out/A", calc, store, numpy.uint32) dl.cleanup = cleanup of = "out/B" def do_write(): assert isinstance(o[0], numpy.core.memmap) write(of, ",".join(str(x) for x in o[0])) append("out/C", "a") ppg.FileGeneratingJob(of, do_write).depends_on(dl) ppg.run_pipegraph() assert read("out/B") == "0,1,2,3,4,5,6,7,8,9" assert read("out/C") == "a" # now, no run... new_pipegraph.new_pipegraph() dl = ppg.MemMappedDataLoadingJob("out/A", calc, store, numpy.uint32) dl.cleanup = cleanup ppg.FileGeneratingJob(of, do_write).depends_on(dl) ppg.run_pipegraph() assert read("out/C") == "a" new_pipegraph.new_pipegraph() def calc2(): append("out/D", "a") return numpy.array(range(0, 12), dtype=numpy.uint32) dl = ppg.MemMappedDataLoadingJob("out/A", calc2, store, numpy.uint32) ppg.FileGeneratingJob(of, do_write).depends_on(dl) dl.cleanup = cleanup ppg.run_pipegraph() assert read("out/D") == "a" assert read("out/B") == "0,1,2,3,4,5,6,7,8,9,10,11" assert read("out/C") == "aa"
def test_raises_on_wrong_dtype(self): import numpy o = [] def calc(): return numpy.array(range(0, 10), dtype=numpy.float) def store(value): o.append(value) def cleanup(): del o[0] dl = ppg.MemMappedDataLoadingJob("out/A", calc, store, numpy.uint32) dl.cleanup = cleanup of = "out/B" def do_write(): assert isinstance(o[0], numpy.core.memmap) write(of, ",".join(str(x) for x in o[0])) ppg.FileGeneratingJob(of, do_write).depends_on(dl) def inner(): ppg.run_pipegraph() assertRaises(ppg.RuntimeError, inner) assert isinstance(dl.lfg.exception, ppg.JobContractError) assert dl.failed
def test_simple(self, new_pipegraph): new_pipegraph.new_pipegraph() import numpy o = [] def calc(): return numpy.array(range(0, 10), dtype=numpy.uint32) def store(value): o.append(value) def cleanup(): del o[0] dl = ppg.MemMappedDataLoadingJob("out/A", calc, store, numpy.uint32) dl.cleanup = cleanup of = "out/B" def do_write(): assert isinstance(o[0], numpy.core.memmap) write(of, ",".join(str(x) for x in o[0])) ppg.FileGeneratingJob(of, do_write).depends_on(dl) dl.depends_on_params(123) ppg.run_pipegraph() assert read("out/B") == "0,1,2,3,4,5,6,7,8,9"
def test_ignore_code_changes(self, new_pipegraph): import numpy import pathlib o = [] def calc(): return numpy.array(range(0, 10), dtype=numpy.uint32) def store(value): o.append(value) def cleanup(): o.clear() dl = ppg.MemMappedDataLoadingJob("out/A", calc, store, numpy.uint32) dl.cleanup = cleanup of = "out/B" def do_write(): assert isinstance(o[0], numpy.core.memmap) write(of, ",".join(str(x) for x in o[0])) ppg.FileGeneratingJob(of, do_write).depends_on(dl) dl.depends_on_params(123) ppg.run_pipegraph() assert read("out/B") == "0,1,2,3,4,5,6,7,8,9" new_pipegraph.new_pipegraph() def calc2(): return numpy.array(range(0, 10), dtype=numpy.uint32) + 1 dl = ppg.MemMappedDataLoadingJob("out/A", calc2, store, numpy.uint32) dl.cleanup = cleanup of = "out/B" ppg.FileGeneratingJob(of, do_write).depends_on(dl) dl.depends_on_params(123) ppg.run_pipegraph() assert read("out/B") == "1,2,3,4,5,6,7,8,9,10" new_pipegraph.new_pipegraph() def calc3(): return numpy.array(range(0, 10), dtype=numpy.uint32) + 1 dl = ppg.MemMappedDataLoadingJob("out/A", calc3, store, numpy.uint32) dl.ignore_code_changes() dl.cleanup = cleanup of = "out/B" ppg.FileGeneratingJob(of, do_write).depends_on(dl) dl.depends_on_params(123) ppg.run_pipegraph() assert read("out/B") == "1,2,3,4,5,6,7,8,9,10" # jup, no rerun new_pipegraph.new_pipegraph() def store2(value): o.append(value) o.append(value) dl = ppg.MemMappedDataLoadingJob("out/A", calc2, store2, numpy.uint32) dl.ignore_code_changes() dl.cleanup = cleanup of = "out/B" ppg.FileGeneratingJob(of, do_write).depends_on(dl) dl.depends_on_params(123) ppg.run_pipegraph() assert read("out/B") == "1,2,3,4,5,6,7,8,9,10" # jup, no rerun new_pipegraph.new_pipegraph() dl = ppg.MemMappedDataLoadingJob("out/A", calc2, store2, numpy.uint32) dl.cleanup = cleanup of = "out/B" ppg.FileGeneratingJob(of, do_write).depends_on(dl) dl.depends_on_params(123) write("out/B", "replace me") ppg.run_pipegraph() assert read("out/B") == "1,2,3,4,5,6,7,8,9,10" # jup, no rerun new_pipegraph.new_pipegraph() dl = ppg.MemMappedDataLoadingJob( pathlib.Path("out/A"), calc2, store2, numpy.uint32 ) dl.cleanup = cleanup of = "out/B" ppg.FileGeneratingJob(of, do_write).depends_on(dl) dl.depends_on_params(123) dl.lfg.depends_on_params( 456 ) # this should trigger just the load invalidation write("out/B", "replace me") ppg.run_pipegraph() assert read("out/B") == "1,2,3,4,5,6,7,8,9,10" # jup, no rerun
def inner(): ppg.MemMappedDataLoadingJob("out/A", calc, store, numpypy.uint32)