def setUp(self): global shelf global shelf2 if shelf == 0: log("Recovery: running test setup...") os.system("rm -Rf /mnt/pmem0/1") os.system("rm -Rf /mnt/pmem0/2") os.system("mkdir -p /mnt/pmem0/1") os.system("mkdir -p /mnt/pmem0/2") shelf = pymm.shelf('myShelf', size_mb=256, load_addr='0x700000000', backend='hstore-cc', pmem_path='/mnt/pmem0/1', force_new=True) shelf2 = pymm.shelf('myShelf-2', size_mb=256, load_addr='0x800000000', backend='hstore-cc', pmem_path='/mnt/pmem0/2', force_new=True) log("Recovery: shelf init OK")
def setUp(self): global force_new self.s = pymm.shelf('myShelf', size_mb=1024, pmem_path='/mnt/pmem0', force_new=force_new) force_new = False
def setUp(self): global force_new self.s = pymm.shelf('myShelf', size_mb=1024, backend="hstore-cc", force_new=force_new) force_new = False
def test_establish(self): log("Testing: establishing shelf and values") shelf = pymm.shelf('myShelfRecovery',size_mb=1024,pmem_path='/mnt/pmem0',force_new=True) # different types shelf.s = 'Hello' shelf.s += ' world!' print(list('Hello')) shelf.f = 1.123 shelf.fm = 2.2 shelf.fm += 1.1 shelf.i = 911 shelf.im = 900 shelf.im += 99 shelf.nd = np.ones((3,)) shelf.nd2 = np.identity(10) shelf.b = b'This is a bytes type' shelf.bm = b'This is a ' shelf.bm += b'modified bytes type' shelf.t = torch.tensor([[1,1,1,1,1],[2,2,2,2,2],[3,3,3,3,3]]) shelf.l = pymm.linked_list() shelf.l.append(1) shelf.l.append(2) shelf.l.append('z') del shelf gc.collect()
def test_recovery(self): log("Testing: recovering shelf and values") shelf = pymm.shelf('myShelfRecovery',pmem_path='/mnt/pmem0',force_new=False) print(">{}<".format(shelf.s)) print(shelf.f) print(shelf.fm) print(shelf.i) print(shelf.im) print(shelf.nd) print(shelf.nd2) print(shelf.b) print(shelf.bm) print(list(shelf.s)) print(round(shelf.fm,2)) print(shelf.t) print(shelf.l) check(shelf.s == 'Hello world!', 'string recovery') check(shelf.f == 1.123, 'float recovery') check(round(shelf.fm,2) == 3.30, 'float modified recovery') check(shelf.i == 911, 'integer recovery') check(shelf.im == 999, 'integer modified recovery') check(np.array_equal(shelf.nd, np.ones((3,))),'1D ndarray') check(np.array_equal(shelf.nd2, np.identity(10)),'2D ndarray') check(shelf.b == b'This is a bytes type', 'bytes') check(shelf.bm == b'This is a modified bytes type', 'modified bytes') check(shelf.t.equal(torch.tensor([[1,1,1,1,1],[2,2,2,2,2],[3,3,3,3,3]])), 'torch tensor') check(shelf.l[0] == 1, 'linked list') check(shelf.l[1] == 2, 'linked list') check(shelf.l[2] == 'z', 'linked list') print(shelf.items) shelf.inspect(verbose=True)
def test_basic_writes(): ''' Test basic array writes ''' import pymm import numpy as np # create new shelf (override any existing myShelf) # s = pymm.shelf('test_basic_writes',32,pmem_path='/mnt/pmem0',force_new=True) # create variable x on shelf (using shadow type) s.x = pymm.ndarray((1000,1000),dtype=np.uint8) if s.x.shape != (1000,1000): raise RuntimeException('demo: s.x.shape check failed') s.x[0] = 1 if s.x[0] != 1: raise('Test 0: failure') if s.x[0][0] != 1: raise('Test 0: failure') s.x[0][0] = 2 if s.x[0][0] != 2: raise('Test 0: failure') print('Test 0 OK!')
def test_dram_mapstore_rcalb(self): log("Running shelf with mapstore and rcalb MM plugin ...") s = pymm.shelf('myShelf4',size_mb=8,backend='mapstore',mm_plugin='libmm-plugin-rcalb.so') s.x = pymm.ndarray((10,10,)) s.y = pymm.ndarray((100,100,)) print(s.items) log("OK!")
def setUp(self): global shelf global shelf2 if shelf == 0: log("Recovery: running test setup...") shelf = pymm.shelf('myShelf', load_addr='0x700000000', backend='hstore-cc', pmem_path='/mnt/pmem0/1', force_new=False) shelf2 = pymm.shelf('myShelf-2', load_addr='0x800000000', backend='hstore-cc', pmem_path='/mnt/pmem0/2', force_new=False) print("Shelf.items = {}".format(shelf.items)) print("Shelf2.items = {}".format(shelf2.items)) log("Recovery: shelf init OK")
def setUp(self): global force_new self.s = pymm.shelf('myShelf', size_mb=1024, pmem_path='/mnt/pmem0', force_new=force_new) pymm.pymmcore.enable_transient_memory(backing_directory='/tmp', pmem_file='/mnt/pmem0/swap', pmem_file_size_gb=1)
def test_dlpack_array(self): log("Testing: dlpack_array ...") shelf = pymm.shelf('myShelf',size_mb=1024,pmem_path='/mnt/pmem0',force_new=True) shelf.a = pymm.dlpack_array((5,),dtype=np.float64) print(shelf.a.addr) print(shelf.a) shelf.inspect() del shelf gc.collect()
def demo(force_new=True): ''' Demonstration of pymm features ''' import pymm import numpy as np # create new shelf (override any existing myShelf) # s = pymm.shelf('myShelf', 1024, pmem_path='/mnt/pmem0', force_new=force_new) # create variable x on shelf (using shadow type) s.x = pymm.ndarray((1000, 1000), dtype=np.float) if s.x.shape != (1000, 1000): raise RuntimeException('demo: s.x.shape check failed') # perform in-place (on-shelf) operations s.x.fill(3) s.x += 2 x_checksum = sum(s.x.tobytes()) # get checksum # write binary array data to file dfile = open("array.dat", "wb") dfile.write(s.x.tobytes()) dfile.close() # create new instance s.z = np.ndarray((1000, 1000), dtype=np.float) # zero-copy read into instance from file with open("array.dat", "rb") as source: source.readinto(memoryview(s.z)) z_checksum = sum(s.z.tobytes()) # get checksum if z_checksum != x_checksum: raise RuntimeError('data checksum mismatch') # this will create a persistent memory copy from RHS DRAM/volatile instance # the right hand side will be garbage collected from skimage import data, io s.i = data.camera() s.j = data.brick() s.blended = s.i + (0.5 * s.j) io.imshow(s.blended) io.show() # remove objects from shelf for item in s.items: s.erase(item) return
def test_hstore_cc(self): log("Running shelf with hstore-cc ...") pmem_path="%s/test_hstore_cc" % (self.pmem_root,) os.system("rm -Rf %s" % (pmem_path,)) os.mkdir(pmem_path) s = pymm.shelf('myShelf5',size_mb=8,backend='hstore-cc',pmem_path=pmem_path) s.x = pymm.ndarray((10,10,)) s.y = pymm.ndarray((100,100,)) print(s.items) log("OK!")
def test_hstore_mm_rcalb(self): log("Running shelf with hstore-mm and rcalb MM plugin ...") pmem_path="%s/test_hstore_mm_rcalb" % (self.pmem_root,) os.system("rm -Rf %s" % (pmem_path,)) os.mkdir(pmem_path) s = pymm.shelf('myShelf6',size_mb=8,backend='hstore-mm',pmem_path=pmem_path,mm_plugin='libmm-plugin-rcalb.so') s.x = pymm.ndarray((10,10,)) s.y = pymm.ndarray((100,100,)) print(s.items) log("OK!")
def test_hstore_mm_default(self): log("Running shelf with hstore-mm and default MM plugin ...") pmem_path="%s/test_hstore_mm_default" % (self.pmem_root,) os.system("rm -Rf %s" % (pmem_path,)) os.mkdir(pmem_path) s = pymm.shelf('myShelf_mm_default',size_mb=8,backend='hstore-mm',pmem_path=pmem_path) s.x = pymm.ndarray((10,10,)) s.y = pymm.ndarray((100,100,)) print(s.items) log("OK!")
def test_dlpack_array_from_dlpack_tf(self): log("Testing: dlpack_array pycapsule generation ...") shelf = pymm.shelf('myShelf', pmem_path='/mnt/pmem0',force_new=False) capsule = shelf.a.as_capsule() print(capsule) log("Testing: dlpack_array importing to tensorflow ...") tf_tensor = tf.experimental.dlpack.from_dlpack(capsule) print(tf_tensor) shelf.inspect() del capsule # del tf_tensor gc.collect() log("Testing: dlpack_array pycapsule tests done")
def XXtest_dlpack_array_from_dlpack_torch(self): log("Testing: dlpack_array pycapsule generation ...") shelf = pymm.shelf('myShelf', pmem_path='/mnt/pmem0',force_new=False) capsule = shelf.a.as_capsule() print(capsule) log("Testing: dlpack_array importing to pytorch ...") torch_tensor = torch.utils.dlpack.from_dlpack(capsule) print(torch_tensor) shelf.inspect() del capsule # del tf_tensor gc.collect() log("Testing: dlpack_array pycapsule tests done")
def test_shelf_transaction(self): shelf = pymm.shelf('myTransactionsShelf', size_mb=1024, pmem_path='/mnt/pmem0', force_new=True) shelf.n = pymm.ndarray((100, 100), dtype=np.uint8) shelf.m = pymm.ndarray((100, 100), dtype=np.uint8) print(shelf.tx_begin([shelf.m, shelf.n])) for i in np.arange(0, 10): shelf.n += 1 shelf.m += 3 print(shelf.items) shelf.inspect(verbose=False) shelf.tx_end() shelf.inspect(verbose=False)
def test_write_operations(): ''' Test write operations ''' import pymm import numpy as np # create new shelf (override any existing myShelf) # s = pymm.shelf('test_write_operations',32,pmem_path='/mnt/pmem0',force_new=True) # create variable x on shelf (using shadow type) s.x = pymm.ndarray((10,10),dtype=np.uint8) s.x.fill(1) # do not modify but make copies -s.x s.x+8 if s.x[0][0] != 1: raise RuntimeError('test failed unexpectedly') return s
def Xtest_transactions(self): shelf = pymm.shelf('myTransactionsShelf', size_mb=1024, pmem_path='/mnt/pmem0', force_new=True) log("Testing: transaction on matrix fill ...") shelf.n = pymm.ndarray((100, 100), dtype=np.uint8) shelf.n += 1 shelf.n += 1 # shelf.s = 'This is a string!' # shelf.f = 1.123 # shelf.i = 645338 # shelf.b = b'Hello' # shelf.w = np.ndarray((100,100),dtype=np.uint8) # shelf.w.fill(ord('a')) # shelf.t = pymm.torch_tensor(np.arange(0,10)) print(shelf.items) shelf.inspect(verbose=False) shelf.persist() shelf.inspect(verbose=False)
def print_error(*args): print(colored(255, 0, 0, *args)) def log(*args): print(colored(0, 255, 255, *args)) print('[TEST]: enabling transient memory ...') pymm.enable_transient_memory(pmem_file='/mnt/pmem0/swap', pmem_file_size_gb=2, backing_directory='/tmp') s = pymm.shelf('myShelf', size_mb=2048, backend="hstore-cc", force_new=True) # create a large right-hand side expression which will be evaluated in pmem transient memory w = np.ndarray((1000000000 * 1), dtype=np.uint8) # 1GB # create something even larger that will fail in pmem and drop into mmap filed w2 = np.ndarray((1000000000 * 10), dtype=np.uint8) # 10GB # copy to shelf s.w = w # force clean up of w del w del w2 gc.collect() gc.get_objects()
from skimage.filters import sobel from skimage.segmentation import felzenszwalb, slic, quickshift, watershed from skimage.util import img_as_float from skimage.segmentation import mark_boundaries segments = slic(image, n_segments=20, compactness=10, sigma=1, start_label=1) return mark_boundaries(image, segments) #-- main -- shelf = pymm.shelf('imageExample', size_mb=32, backend='hstore-cc', pmem_path='/mnt/pmem0') # load image to persistent shelf # if 'testImage' not in shelf.items: print('Loading test image from file...') shelf.testImage = data.coins() # create segmented image on shelf # if 'segmentedTestImage' not in shelf.items: shelf.segmentedTestImage = perform_segmentation(shelf.testImage) # display segmented image #
def test_default(self): log("Running shelf with default backend ...") s = pymm.shelf('myShelf',size_mb=8,pmem_path=self.pmem_root,force_new=True) s.items log("OK!")
def setUp(self): global force_new self.s = pymm.shelf('myShelf', size_mb=1024, backend="mapstore")
# Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # http://www.apache.org/licenses/LICENSE-2.0 # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import pymm import numpy as np # create new shelf (override any existing myShelf) # s = pymm.shelf('myShelf', '/mnt/pmem0', 1024, force_new=True) print("Created shelf OK.") # create variable x on shelf (using shadow type) s.x = np.random.uniform(low=-1.0, high=1.0, size=100000000) print("Created array 'x' on shelf OK. Data at {}".format(s.x.addr)) # in-place random initialization (could be faster with vectorize + copy) print(s.x) # sort in-place s.x.sort() print("Sorted array 'x' on shelf OK. Data at {}".format(s.x.addr)) print(s.x) print("Use s and s.x to access shelf...")
def main() -> None: script_start_time = time.time() parser = argparse.ArgumentParser() parser.add_argument("model", type=str, choices=sorted(model_map.keys()), help="the model type") parser.add_argument("-K", "--posterior_sample_size", type=int, default=np.inf, help="the number of samples posterior expects") parser.add_argument("-b", "--batch_size", type=int, default=100, help="batch size") parser.add_argument("-n", "--learning_rate", type=float, default=1e-4, help="learning rate") parser.add_argument("-m", "--momentum", type=float, default=0, help="sgd momentum") parser.add_argument("-e", "--epochs", type=int, default=int(1e6), help="num epochs") parser.add_argument("-p", "--path", type=str, default="/scratch/aewood/data/mnist") parser.add_argument("-c", "--cuda", type=int, default=-1, help="gpu id (-1 for cpu)") parser.add_argument("-s", "--size_mb", type=int, default=40000, help="size of shelf in mb") parser.add_argument("-f", "--shelf_file", type=str, default="/mnt/pmem0", help="pymm shelf directory") parser.add_argument("-r", "--bpost", type=int, default=1) parser.add_argument("-back", "--backend", type=str, default="hstore-cc", choices=["hstore-cc", "mapstore"], help="The backend of the shelf - hstore-cc/ mapstore") parser.add_argument("-csv", "--results_filepath", type=str, default="./results/mnist/pymm_timings.csv") args = parser.parse_args() if not os.path.exists(args.path): os.makedirs(args.path) results_dir: str = os.path.abspath(os.path.dirname(args.results_filepath)) if not os.path.exists(results_dir): os.makedirs(results_dir) start_time: float = time.time() train_loader = pt.utils.data.DataLoader( ptv.datasets.MNIST( args.path, train=True, download=True, transform=ptv.transforms.ToTensor() # transform=ptv.transforms.Compose([ # ptv.transforms.ToTensor(), # ptv.transforms.Normalize((0.1307,), # (0.3081,)) # ]) ), batch_size=args.batch_size, shuffle=True) end_train_loader_time = time.time() - start_time start_time = time.time() test_loader = pt.utils.data.DataLoader( ptv.datasets.MNIST( args.path, train=False, download=True, transform=ptv.transforms.ToTensor() # transform=ptv.transforms.Compose([ # ptv.transforms.ToTensor(), # ptv.transforms.Normalize((0.1307,), # (0.3081,)) # ]) ), batch_size=args.batch_size, shuffle=True) end_test_loader_time = time.time() - start_time if np.isinf(args.posterior_sample_size): args.posterior_sample_size = len(train_loader) * args.epochs start_time = time.time() shelf = pymm.shelf("mnist_pymm_posterior", size_mb=args.size_mb, pmem_path=args.shelf_file, backend=args.backend, force_new=True) end_shelf_time = time.time() - start_time print("loading model") start_time = time.time() m = model_map[args.model]() if args.cuda > -1: m = m.to(args.cuda) optimizer = pt.optim.SGD(m.parameters(), lr=args.learning_rate, momentum=args.momentum) end_make_model_time = time.time() - start_time num_params: int = m.get_params().shape[0] start_time = time.time() posterior = LazyPymmPosterior(num_params, shelf, K=args.posterior_sample_size) end_make_posterior_time = time.time() - start_time print("num params: %s" % num_params, "posterior will take %s bytes" % posterior.nbytes) epoch_times: List[float] = list() start_experiment_time = time.time() # update posterior with first parameter sample posterior.update(m.get_params()) for e in range(args.epochs): start_epoch_time = time.time() train_one_epoch(m, optimizer, train_loader, e + 1, posterior, args.bpost, args.cuda) epoch_times.append(time.time() - start_epoch_time) print("finalizing posterior") start_finalize_time = time.time() posterior.finalize() end_finalize_time = time.time() - start_finalize_time print("done") end_experiment_time = time.time() - start_experiment_time end_script_time = time.time() - script_start_time # posterior.sample() with open(args.results_filepath, "w") as f: writer = csv.writer(f, delimiter=",") writer.writerow(["posterior size (byte)", posterior.nbytes]) writer.writerow(["train data loading time (s)", end_train_loader_time]) writer.writerow(["test data loading time (s)", end_test_loader_time]) writer.writerow(["open shelf time (s)", end_shelf_time]) writer.writerow(["make model time (s)", end_make_model_time]) writer.writerow(["make posterior time (s)", end_make_posterior_time]) for e, t in enumerate(epoch_times): writer.writerow(["epoch %s time (s)" % e, t]) writer.writerow(["posterior finalize time (s)", end_finalize_time]) writer.writerow(["total experiment time (s)", end_experiment_time]) writer.writerow(["total script time (s)", end_script_time])
import unittest import pymm import numpy as np def colored(r, g, b, text): return "\033[38;2;{};{};{}m{} \033[38;2;255;255;255m".format(r, g, b, text) def log(*args): print(colored(0, 255, 255, *args)) shelf = pymm.shelf('myShelf', size_mb=1024, pmem_path='/mnt/pmem0', backend="hstore-cc", force_new=True) class TestLinkedList(unittest.TestCase): def test_A_list_construction(self): global shelf log("Testing: pymm.linked_list construction") shelf.x = pymm.linked_list() print(shelf.x) def test_B_list_append(self): global shelf log("Testing: pymm.linked_list append method") shelf.x.append(123) # will be stored inline
# Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # http://www.apache.org/licenses/LICENSE-2.0 # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # import pymm import numpy as np # create new shelf (override any existing myShelf) # s = pymm.shelf('myShelf', 1024, pmem_path='/mnt/pmem0', force_new=True) print("Created shelf OK.") # create variable x on shelf (using shadow type) s.x = np.random.uniform(low=-1.0, high=1.0, size=100000000) print("Created array 'x' on shelf OK. Data at {}".format(s.x.addr)) # in-place random initialization (could be faster with vectorize + copy) print(s.x) # sort in-place s.x.sort() print("Sorted array 'x' on shelf OK. Data at {}".format(s.x.addr)) print(s.x) print("Use s and s.x to access shelf...")
def colored(r, g, b, text): return "\033[38;2;{};{};{}m{} \033[38;2;255;255;255m".format(r, g, b, text) def print_error(*args): print(colored(255, 0, 0, *args)) def log(*args): print(colored(0, 255, 255, *args)) shelf = pymm.shelf('myShelf', size_mb=1024, pmem_path='/mnt/pmem0', force_new=True) class TestNdarray(unittest.TestCase): def test_integer_assignment(self): log("Test: integer assignment and re-assignment") shelf.a = 2 shelf.b = 30 shelf.b = shelf.b / shelf.a print(shelf.b) shelf.erase('a') shelf.erase('b') def test_integer_with_ndarray(self): log("Test: array change with integer")
def setUp(self): self.s = pymm.shelf('myShelf',size_mb=1024,backend='hstore-cc',pmem_path='/dev/dax1.0',force_new=True) print(self.s.items)
def test_dram_mapstore(self): log("Running shelf with mapstore and default MM plugin ...") s = pymm.shelf('myShelf2',size_mb=8,backend='mapstore') # note, no force_new or pmem_path s.x = pymm.ndarray((10,10,)) print(s.items) log("OK!")