def setUp(self):
global shelf
global shelf2
if shelf == 0:
log("Recovery: running test setup...")
os.system("rm -Rf /mnt/pmem0/1")
os.system("rm -Rf /mnt/pmem0/2")
os.system("mkdir -p /mnt/pmem0/1")
os.system("mkdir -p /mnt/pmem0/2")
shelf = pymm.shelf('myShelf',
size_mb=256,
load_addr='0x700000000',
backend='hstore-cc',
pmem_path='/mnt/pmem0/1',
force_new=True)
shelf2 = pymm.shelf('myShelf-2',
size_mb=256,
load_addr='0x800000000',
backend='hstore-cc',
pmem_path='/mnt/pmem0/2',
force_new=True)
log("Recovery: shelf init OK")
def setUp(self):
global force_new
self.s = pymm.shelf('myShelf',
size_mb=1024,
pmem_path='/mnt/pmem0',
force_new=force_new)
force_new = False
def setUp(self):
global force_new
self.s = pymm.shelf('myShelf',
size_mb=1024,
backend="hstore-cc",
force_new=force_new)
force_new = False
def test_establish(self):
log("Testing: establishing shelf and values")
shelf = pymm.shelf('myShelfRecovery',size_mb=1024,pmem_path='/mnt/pmem0',force_new=True)
# different types
shelf.s = 'Hello'
shelf.s += ' world!'
print(list('Hello'))
shelf.f = 1.123
shelf.fm = 2.2
shelf.fm += 1.1
shelf.i = 911
shelf.im = 900
shelf.im += 99
shelf.nd = np.ones((3,))
shelf.nd2 = np.identity(10)
shelf.b = b'This is a bytes type'
shelf.bm = b'This is a '
shelf.bm += b'modified bytes type'
shelf.t = torch.tensor([[1,1,1,1,1],[2,2,2,2,2],[3,3,3,3,3]])
shelf.l = pymm.linked_list()
shelf.l.append(1)
shelf.l.append(2)
shelf.l.append('z')
del shelf
gc.collect()
def test_recovery(self):
log("Testing: recovering shelf and values")
shelf = pymm.shelf('myShelfRecovery',pmem_path='/mnt/pmem0',force_new=False)
print(">{}<".format(shelf.s))
print(shelf.f)
print(shelf.fm)
print(shelf.i)
print(shelf.im)
print(shelf.nd)
print(shelf.nd2)
print(shelf.b)
print(shelf.bm)
print(list(shelf.s))
print(round(shelf.fm,2))
print(shelf.t)
print(shelf.l)
check(shelf.s == 'Hello world!', 'string recovery')
check(shelf.f == 1.123, 'float recovery')
check(round(shelf.fm,2) == 3.30, 'float modified recovery')
check(shelf.i == 911, 'integer recovery')
check(shelf.im == 999, 'integer modified recovery')
check(np.array_equal(shelf.nd, np.ones((3,))),'1D ndarray')
check(np.array_equal(shelf.nd2, np.identity(10)),'2D ndarray')
check(shelf.b == b'This is a bytes type', 'bytes')
check(shelf.bm == b'This is a modified bytes type', 'modified bytes')
check(shelf.t.equal(torch.tensor([[1,1,1,1,1],[2,2,2,2,2],[3,3,3,3,3]])), 'torch tensor')
check(shelf.l[0] == 1, 'linked list')
check(shelf.l[1] == 2, 'linked list')
check(shelf.l[2] == 'z', 'linked list')
print(shelf.items)
shelf.inspect(verbose=True)
def test_basic_writes():
'''
Test basic array writes
'''
import pymm
import numpy as np
# create new shelf (override any existing myShelf)
#
s = pymm.shelf('test_basic_writes',32,pmem_path='/mnt/pmem0',force_new=True)
# create variable x on shelf (using shadow type)
s.x = pymm.ndarray((1000,1000),dtype=np.uint8)
if s.x.shape != (1000,1000):
raise RuntimeException('demo: s.x.shape check failed')
s.x[0] = 1
if s.x[0] != 1:
raise('Test 0: failure')
if s.x[0][0] != 1:
raise('Test 0: failure')
s.x[0][0] = 2
if s.x[0][0] != 2:
raise('Test 0: failure')
print('Test 0 OK!')
def test_dram_mapstore_rcalb(self):
log("Running shelf with mapstore and rcalb MM plugin ...")
s = pymm.shelf('myShelf4',size_mb=8,backend='mapstore',mm_plugin='libmm-plugin-rcalb.so')
s.x = pymm.ndarray((10,10,))
s.y = pymm.ndarray((100,100,))
print(s.items)
log("OK!")
def setUp(self):
global shelf
global shelf2
if shelf == 0:
log("Recovery: running test setup...")
shelf = pymm.shelf('myShelf', load_addr='0x700000000',
backend='hstore-cc', pmem_path='/mnt/pmem0/1', force_new=False)
shelf2 = pymm.shelf('myShelf-2', load_addr='0x800000000',
backend='hstore-cc', pmem_path='/mnt/pmem0/2', force_new=False)
print("Shelf.items = {}".format(shelf.items))
print("Shelf2.items = {}".format(shelf2.items))
log("Recovery: shelf init OK")
def setUp(self):
global force_new
self.s = pymm.shelf('myShelf',
size_mb=1024,
pmem_path='/mnt/pmem0',
force_new=force_new)
pymm.pymmcore.enable_transient_memory(backing_directory='/tmp',
pmem_file='/mnt/pmem0/swap',
pmem_file_size_gb=1)
def test_dlpack_array(self):
log("Testing: dlpack_array ...")
shelf = pymm.shelf('myShelf',size_mb=1024,pmem_path='/mnt/pmem0',force_new=True)
shelf.a = pymm.dlpack_array((5,),dtype=np.float64)
print(shelf.a.addr)
print(shelf.a)
shelf.inspect()
del shelf
gc.collect()
def demo(force_new=True):
'''
Demonstration of pymm features
'''
import pymm
import numpy as np
# create new shelf (override any existing myShelf)
#
s = pymm.shelf('myShelf',
1024,
pmem_path='/mnt/pmem0',
force_new=force_new)
# create variable x on shelf (using shadow type)
s.x = pymm.ndarray((1000, 1000), dtype=np.float)
if s.x.shape != (1000, 1000):
raise RuntimeException('demo: s.x.shape check failed')
# perform in-place (on-shelf) operations
s.x.fill(3)
s.x += 2
x_checksum = sum(s.x.tobytes()) # get checksum
# write binary array data to file
dfile = open("array.dat", "wb")
dfile.write(s.x.tobytes())
dfile.close()
# create new instance
s.z = np.ndarray((1000, 1000), dtype=np.float)
# zero-copy read into instance from file
with open("array.dat", "rb") as source:
source.readinto(memoryview(s.z))
z_checksum = sum(s.z.tobytes()) # get checksum
if z_checksum != x_checksum:
raise RuntimeError('data checksum mismatch')
# this will create a persistent memory copy from RHS DRAM/volatile instance
# the right hand side will be garbage collected
from skimage import data, io
s.i = data.camera()
s.j = data.brick()
s.blended = s.i + (0.5 * s.j)
io.imshow(s.blended)
io.show()
# remove objects from shelf
for item in s.items:
s.erase(item)
return
def test_hstore_cc(self):
log("Running shelf with hstore-cc ...")
pmem_path="%s/test_hstore_cc" % (self.pmem_root,)
os.system("rm -Rf %s" % (pmem_path,))
os.mkdir(pmem_path)
s = pymm.shelf('myShelf5',size_mb=8,backend='hstore-cc',pmem_path=pmem_path)
s.x = pymm.ndarray((10,10,))
s.y = pymm.ndarray((100,100,))
print(s.items)
log("OK!")
def test_hstore_mm_rcalb(self):
log("Running shelf with hstore-mm and rcalb MM plugin ...")
pmem_path="%s/test_hstore_mm_rcalb" % (self.pmem_root,)
os.system("rm -Rf %s" % (pmem_path,))
os.mkdir(pmem_path)
s = pymm.shelf('myShelf6',size_mb=8,backend='hstore-mm',pmem_path=pmem_path,mm_plugin='libmm-plugin-rcalb.so')
s.x = pymm.ndarray((10,10,))
s.y = pymm.ndarray((100,100,))
print(s.items)
log("OK!")
def test_hstore_mm_default(self):
log("Running shelf with hstore-mm and default MM plugin ...")
pmem_path="%s/test_hstore_mm_default" % (self.pmem_root,)
os.system("rm -Rf %s" % (pmem_path,))
os.mkdir(pmem_path)
s = pymm.shelf('myShelf_mm_default',size_mb=8,backend='hstore-mm',pmem_path=pmem_path)
s.x = pymm.ndarray((10,10,))
s.y = pymm.ndarray((100,100,))
print(s.items)
log("OK!")
def test_dlpack_array_from_dlpack_tf(self):
log("Testing: dlpack_array pycapsule generation ...")
shelf = pymm.shelf('myShelf', pmem_path='/mnt/pmem0',force_new=False)
capsule = shelf.a.as_capsule()
print(capsule)
log("Testing: dlpack_array importing to tensorflow ...")
tf_tensor = tf.experimental.dlpack.from_dlpack(capsule)
print(tf_tensor)
shelf.inspect()
del capsule
# del tf_tensor
gc.collect()
log("Testing: dlpack_array pycapsule tests done")
def XXtest_dlpack_array_from_dlpack_torch(self):
log("Testing: dlpack_array pycapsule generation ...")
shelf = pymm.shelf('myShelf', pmem_path='/mnt/pmem0',force_new=False)
capsule = shelf.a.as_capsule()
print(capsule)
log("Testing: dlpack_array importing to pytorch ...")
torch_tensor = torch.utils.dlpack.from_dlpack(capsule)
print(torch_tensor)
shelf.inspect()
del capsule
# del tf_tensor
gc.collect()
log("Testing: dlpack_array pycapsule tests done")
def test_shelf_transaction(self):
shelf = pymm.shelf('myTransactionsShelf',
size_mb=1024,
pmem_path='/mnt/pmem0',
force_new=True)
shelf.n = pymm.ndarray((100, 100), dtype=np.uint8)
shelf.m = pymm.ndarray((100, 100), dtype=np.uint8)
print(shelf.tx_begin([shelf.m, shelf.n]))
for i in np.arange(0, 10):
shelf.n += 1
shelf.m += 3
print(shelf.items)
shelf.inspect(verbose=False)
shelf.tx_end()
shelf.inspect(verbose=False)
def test_write_operations():
'''
Test write operations
'''
import pymm
import numpy as np
# create new shelf (override any existing myShelf)
#
s = pymm.shelf('test_write_operations',32,pmem_path='/mnt/pmem0',force_new=True)
# create variable x on shelf (using shadow type)
s.x = pymm.ndarray((10,10),dtype=np.uint8)
s.x.fill(1)
# do not modify but make copies
-s.x
s.x+8
if s.x[0][0] != 1:
raise RuntimeError('test failed unexpectedly')
return s
def Xtest_transactions(self):
shelf = pymm.shelf('myTransactionsShelf',
size_mb=1024,
pmem_path='/mnt/pmem0',
force_new=True)
log("Testing: transaction on matrix fill ...")
shelf.n = pymm.ndarray((100, 100), dtype=np.uint8)
shelf.n += 1
shelf.n += 1
# shelf.s = 'This is a string!'
# shelf.f = 1.123
# shelf.i = 645338
# shelf.b = b'Hello'
# shelf.w = np.ndarray((100,100),dtype=np.uint8)
# shelf.w.fill(ord('a'))
# shelf.t = pymm.torch_tensor(np.arange(0,10))
print(shelf.items)
shelf.inspect(verbose=False)
shelf.persist()
shelf.inspect(verbose=False)
def print_error(*args):
print(colored(255, 0, 0, *args))
def log(*args):
print(colored(0, 255, 255, *args))
print('[TEST]: enabling transient memory ...')
pymm.enable_transient_memory(pmem_file='/mnt/pmem0/swap',
pmem_file_size_gb=2,
backing_directory='/tmp')
s = pymm.shelf('myShelf', size_mb=2048, backend="hstore-cc", force_new=True)
# create a large right-hand side expression which will be evaluated in pmem transient memory
w = np.ndarray((1000000000 * 1), dtype=np.uint8) # 1GB
# create something even larger that will fail in pmem and drop into mmap filed
w2 = np.ndarray((1000000000 * 10), dtype=np.uint8) # 10GB
# copy to shelf
s.w = w
# force clean up of w
del w
del w2
gc.collect()
gc.get_objects()
from skimage.filters import sobel
from skimage.segmentation import felzenszwalb, slic, quickshift, watershed
from skimage.util import img_as_float
from skimage.segmentation import mark_boundaries
segments = slic(image,
n_segments=20,
compactness=10,
sigma=1,
start_label=1)
return mark_boundaries(image, segments)
#-- main --
shelf = pymm.shelf('imageExample',
size_mb=32,
backend='hstore-cc',
pmem_path='/mnt/pmem0')
# load image to persistent shelf
#
if 'testImage' not in shelf.items:
print('Loading test image from file...')
shelf.testImage = data.coins()
# create segmented image on shelf
#
if 'segmentedTestImage' not in shelf.items:
shelf.segmentedTestImage = perform_segmentation(shelf.testImage)
# display segmented image
#
def test_default(self):
log("Running shelf with default backend ...")
s = pymm.shelf('myShelf',size_mb=8,pmem_path=self.pmem_root,force_new=True)
s.items
log("OK!")
def setUp(self):
global force_new
self.s = pymm.shelf('myShelf', size_mb=1024, backend="mapstore")
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import pymm
import numpy as np
# create new shelf (override any existing myShelf)
#
s = pymm.shelf('myShelf', '/mnt/pmem0', 1024, force_new=True)
print("Created shelf OK.")
# create variable x on shelf (using shadow type)
s.x = np.random.uniform(low=-1.0, high=1.0, size=100000000)
print("Created array 'x' on shelf OK. Data at {}".format(s.x.addr))
# in-place random initialization (could be faster with vectorize + copy)
print(s.x)
# sort in-place
s.x.sort()
print("Sorted array 'x' on shelf OK. Data at {}".format(s.x.addr))
print(s.x)
print("Use s and s.x to access shelf...")
def main() -> None:
script_start_time = time.time()
parser = argparse.ArgumentParser()
parser.add_argument("model",
type=str,
choices=sorted(model_map.keys()),
help="the model type")
parser.add_argument("-K",
"--posterior_sample_size",
type=int,
default=np.inf,
help="the number of samples posterior expects")
parser.add_argument("-b",
"--batch_size",
type=int,
default=100,
help="batch size")
parser.add_argument("-n",
"--learning_rate",
type=float,
default=1e-4,
help="learning rate")
parser.add_argument("-m",
"--momentum",
type=float,
default=0,
help="sgd momentum")
parser.add_argument("-e",
"--epochs",
type=int,
default=int(1e6),
help="num epochs")
parser.add_argument("-p",
"--path",
type=str,
default="/scratch/aewood/data/mnist")
parser.add_argument("-c",
"--cuda",
type=int,
default=-1,
help="gpu id (-1 for cpu)")
parser.add_argument("-s",
"--size_mb",
type=int,
default=40000,
help="size of shelf in mb")
parser.add_argument("-f",
"--shelf_file",
type=str,
default="/mnt/pmem0",
help="pymm shelf directory")
parser.add_argument("-r", "--bpost", type=int, default=1)
parser.add_argument("-back",
"--backend",
type=str,
default="hstore-cc",
choices=["hstore-cc", "mapstore"],
help="The backend of the shelf - hstore-cc/ mapstore")
parser.add_argument("-csv",
"--results_filepath",
type=str,
default="./results/mnist/pymm_timings.csv")
args = parser.parse_args()
if not os.path.exists(args.path):
os.makedirs(args.path)
results_dir: str = os.path.abspath(os.path.dirname(args.results_filepath))
if not os.path.exists(results_dir):
os.makedirs(results_dir)
start_time: float = time.time()
train_loader = pt.utils.data.DataLoader(
ptv.datasets.MNIST(
args.path,
train=True,
download=True,
transform=ptv.transforms.ToTensor()
# transform=ptv.transforms.Compose([
# ptv.transforms.ToTensor(),
# ptv.transforms.Normalize((0.1307,),
# (0.3081,))
# ])
),
batch_size=args.batch_size,
shuffle=True)
end_train_loader_time = time.time() - start_time
start_time = time.time()
test_loader = pt.utils.data.DataLoader(
ptv.datasets.MNIST(
args.path,
train=False,
download=True,
transform=ptv.transforms.ToTensor()
# transform=ptv.transforms.Compose([
# ptv.transforms.ToTensor(),
# ptv.transforms.Normalize((0.1307,),
# (0.3081,))
# ])
),
batch_size=args.batch_size,
shuffle=True)
end_test_loader_time = time.time() - start_time
if np.isinf(args.posterior_sample_size):
args.posterior_sample_size = len(train_loader) * args.epochs
start_time = time.time()
shelf = pymm.shelf("mnist_pymm_posterior",
size_mb=args.size_mb,
pmem_path=args.shelf_file,
backend=args.backend,
force_new=True)
end_shelf_time = time.time() - start_time
print("loading model")
start_time = time.time()
m = model_map[args.model]()
if args.cuda > -1:
m = m.to(args.cuda)
optimizer = pt.optim.SGD(m.parameters(),
lr=args.learning_rate,
momentum=args.momentum)
end_make_model_time = time.time() - start_time
num_params: int = m.get_params().shape[0]
start_time = time.time()
posterior = LazyPymmPosterior(num_params,
shelf,
K=args.posterior_sample_size)
end_make_posterior_time = time.time() - start_time
print("num params: %s" % num_params,
"posterior will take %s bytes" % posterior.nbytes)
epoch_times: List[float] = list()
start_experiment_time = time.time()
# update posterior with first parameter sample
posterior.update(m.get_params())
for e in range(args.epochs):
start_epoch_time = time.time()
train_one_epoch(m, optimizer, train_loader, e + 1, posterior,
args.bpost, args.cuda)
epoch_times.append(time.time() - start_epoch_time)
print("finalizing posterior")
start_finalize_time = time.time()
posterior.finalize()
end_finalize_time = time.time() - start_finalize_time
print("done")
end_experiment_time = time.time() - start_experiment_time
end_script_time = time.time() - script_start_time
# posterior.sample()
with open(args.results_filepath, "w") as f:
writer = csv.writer(f, delimiter=",")
writer.writerow(["posterior size (byte)", posterior.nbytes])
writer.writerow(["train data loading time (s)", end_train_loader_time])
writer.writerow(["test data loading time (s)", end_test_loader_time])
writer.writerow(["open shelf time (s)", end_shelf_time])
writer.writerow(["make model time (s)", end_make_model_time])
writer.writerow(["make posterior time (s)", end_make_posterior_time])
for e, t in enumerate(epoch_times):
writer.writerow(["epoch %s time (s)" % e, t])
writer.writerow(["posterior finalize time (s)", end_finalize_time])
writer.writerow(["total experiment time (s)", end_experiment_time])
writer.writerow(["total script time (s)", end_script_time])
import unittest
import pymm
import numpy as np
def colored(r, g, b, text):
return "\033[38;2;{};{};{}m{} \033[38;2;255;255;255m".format(r, g, b, text)
def log(*args):
print(colored(0, 255, 255, *args))
shelf = pymm.shelf('myShelf',
size_mb=1024,
pmem_path='/mnt/pmem0',
backend="hstore-cc",
force_new=True)
class TestLinkedList(unittest.TestCase):
def test_A_list_construction(self):
global shelf
log("Testing: pymm.linked_list construction")
shelf.x = pymm.linked_list()
print(shelf.x)
def test_B_list_append(self):
global shelf
log("Testing: pymm.linked_list append method")
shelf.x.append(123) # will be stored inline
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import pymm
import numpy as np
# create new shelf (override any existing myShelf)
#
s = pymm.shelf('myShelf', 1024, pmem_path='/mnt/pmem0', force_new=True)
print("Created shelf OK.")
# create variable x on shelf (using shadow type)
s.x = np.random.uniform(low=-1.0, high=1.0, size=100000000)
print("Created array 'x' on shelf OK. Data at {}".format(s.x.addr))
# in-place random initialization (could be faster with vectorize + copy)
print(s.x)
# sort in-place
s.x.sort()
print("Sorted array 'x' on shelf OK. Data at {}".format(s.x.addr))
print(s.x)
print("Use s and s.x to access shelf...")
def colored(r, g, b, text):
return "\033[38;2;{};{};{}m{} \033[38;2;255;255;255m".format(r, g, b, text)
def print_error(*args):
print(colored(255, 0, 0, *args))
def log(*args):
print(colored(0, 255, 255, *args))
shelf = pymm.shelf('myShelf',
size_mb=1024,
pmem_path='/mnt/pmem0',
force_new=True)
class TestNdarray(unittest.TestCase):
def test_integer_assignment(self):
log("Test: integer assignment and re-assignment")
shelf.a = 2
shelf.b = 30
shelf.b = shelf.b / shelf.a
print(shelf.b)
shelf.erase('a')
shelf.erase('b')
def test_integer_with_ndarray(self):
log("Test: array change with integer")
def setUp(self):
self.s = pymm.shelf('myShelf',size_mb=1024,backend='hstore-cc',pmem_path='/dev/dax1.0',force_new=True)
print(self.s.items)
def test_dram_mapstore(self):
log("Running shelf with mapstore and default MM plugin ...")
s = pymm.shelf('myShelf2',size_mb=8,backend='mapstore') # note, no force_new or pmem_path
s.x = pymm.ndarray((10,10,))
print(s.items)
log("OK!")
