def __init__(self,
num_params: int,
shelf,
K: int = 15,
dtype: np.dtype = np.float32) -> None:
super().__init__(num_params, dtype=dtype)
self.shelf = shelf
self.shelf.mu: pymm.ndarray = pymm.ndarray((self.num_params, 1),
dtype=self.dtype)
self.shelf.sec_moment_uncentered: pymm.ndarray = pymm.ndarray(
(self.num_params, 1), dtype=self.dtype)
self.shelf.diag: pymm.ndarray = pymm.ndarray((self.num_params, 1),
dtype=self.dtype)
# self.shelf.cov = self.shelf.ndarray((self.num_params, self.num_params),
# dtype=self.dtype)
self.shelf.D_hat: pymm.ndarray = pymm.ndarray((self.num_params, K),
dtype=self.dtype)
# IMPORTANT TO STORE INT CONSTS IN DRAM
# AT LEAST UNTIL PYMM INTS CAN BE USED AS INDICES INTO PYMM ARRAYS
self.shelf.K: pymm.integer_value = K
self.shelf.num_samples: pymm.integer_value = 0
self.D_hat_idx: int = 0
self.shelf.mu.fill(0)
self.shelf.sec_moment_uncentered.fill(0)
self.shelf.diag.fill(0)
self.shelf.D_hat.fill(0)
def test_dram_mapstore_rcalb(self):
log("Running shelf with mapstore and rcalb MM plugin ...")
s = pymm.shelf('myShelf4',size_mb=8,backend='mapstore',mm_plugin='libmm-plugin-rcalb.so')
s.x = pymm.ndarray((10,10,))
s.y = pymm.ndarray((100,100,))
print(s.items)
log("OK!")
def test_hstore_mm_default(self):
log("Running shelf with hstore-mm and default MM plugin ...")
pmem_path="%s/test_hstore_mm_default" % (self.pmem_root,)
os.system("rm -Rf %s" % (pmem_path,))
os.mkdir(pmem_path)
s = pymm.shelf('myShelf_mm_default',size_mb=8,backend='hstore-mm',pmem_path=pmem_path)
s.x = pymm.ndarray((10,10,))
s.y = pymm.ndarray((100,100,))
print(s.items)
log("OK!")
def test_hstore_cc(self):
log("Running shelf with hstore-cc ...")
pmem_path="%s/test_hstore_cc" % (self.pmem_root,)
os.system("rm -Rf %s" % (pmem_path,))
os.mkdir(pmem_path)
s = pymm.shelf('myShelf5',size_mb=8,backend='hstore-cc',pmem_path=pmem_path)
s.x = pymm.ndarray((10,10,))
s.y = pymm.ndarray((100,100,))
print(s.items)
log("OK!")
def test_hstore_mm_rcalb(self):
log("Running shelf with hstore-mm and rcalb MM plugin ...")
pmem_path="%s/test_hstore_mm_rcalb" % (self.pmem_root,)
os.system("rm -Rf %s" % (pmem_path,))
os.mkdir(pmem_path)
s = pymm.shelf('myShelf6',size_mb=8,backend='hstore-mm',pmem_path=pmem_path,mm_plugin='libmm-plugin-rcalb.so')
s.x = pymm.ndarray((10,10,))
s.y = pymm.ndarray((100,100,))
print(s.items)
log("OK!")
def test_matrixop_and_reshape(self):
self.s.theta = pymm.ndarray((
3,
4,
))
self.s.theta.fill(2.0)
self.s.moment = pymm.ndarray((
3,
4,
))
self.s.moment.fill(1.2)
self.s.theta_deviation = (self.s.theta - self.s.moment).reshape(-1)
def test_ndarray(self):
log("Testing: ndarray")
self.s.x = pymm.ndarray((100, 100))
n = np.ndarray((100, 100))
# shelf type S
self.assertTrue(
str(type(self.s.x)) == "<class 'pymm.ndarray.shelved_ndarray'>")
self.s.x.fill(1)
# shelf type S after in-place operation
self.assertTrue(
str(type(self.s.x)) == "<class 'pymm.ndarray.shelved_ndarray'>")
# shelf type S * NS (non-shelf type)
self.assertTrue(str(type(self.s.x * n)) == "<class 'numpy.ndarray'>")
# shelf type NS * S
self.assertTrue(str(type(n * self.s.x)) == "<class 'numpy.ndarray'>")
# shelf type S * shelf type S
self.assertTrue(
str(type(self.s.x * self.s.x)) == "<class 'numpy.ndarray'>")
self.s.erase('x')
log("Testing: ndarray OK!")
def test_basic_writes():
'''
Test basic array writes
'''
import pymm
import numpy as np
# create new shelf (override any existing myShelf)
#
s = pymm.shelf('test_basic_writes',32,pmem_path='/mnt/pmem0',force_new=True)
# create variable x on shelf (using shadow type)
s.x = pymm.ndarray((1000,1000),dtype=np.uint8)
if s.x.shape != (1000,1000):
raise RuntimeException('demo: s.x.shape check failed')
s.x[0] = 1
if s.x[0] != 1:
raise('Test 0: failure')
if s.x[0][0] != 1:
raise('Test 0: failure')
s.x[0][0] = 2
if s.x[0][0] != 2:
raise('Test 0: failure')
print('Test 0 OK!')
def XXX_test_B_add_shelf_ndarray(self):
log("Testing: pymm.linked_list: creating an ndarray on shelf, then adding to list"
)
shelf.n = pymm.ndarray((
3,
8,
))
shelf.x.append(shelf.n)
print(shelf.x)
def demo(force_new=True):
'''
Demonstration of pymm features
'''
import pymm
import numpy as np
# create new shelf (override any existing myShelf)
#
s = pymm.shelf('myShelf',
1024,
pmem_path='/mnt/pmem0',
force_new=force_new)
# create variable x on shelf (using shadow type)
s.x = pymm.ndarray((1000, 1000), dtype=np.float)
if s.x.shape != (1000, 1000):
raise RuntimeException('demo: s.x.shape check failed')
# perform in-place (on-shelf) operations
s.x.fill(3)
s.x += 2
x_checksum = sum(s.x.tobytes()) # get checksum
# write binary array data to file
dfile = open("array.dat", "wb")
dfile.write(s.x.tobytes())
dfile.close()
# create new instance
s.z = np.ndarray((1000, 1000), dtype=np.float)
# zero-copy read into instance from file
with open("array.dat", "rb") as source:
source.readinto(memoryview(s.z))
z_checksum = sum(s.z.tobytes()) # get checksum
if z_checksum != x_checksum:
raise RuntimeError('data checksum mismatch')
# this will create a persistent memory copy from RHS DRAM/volatile instance
# the right hand side will be garbage collected
from skimage import data, io
s.i = data.camera()
s.j = data.brick()
s.blended = s.i + (0.5 * s.j)
io.imshow(s.blended)
io.show()
# remove objects from shelf
for item in s.items:
s.erase(item)
return
def test_ndarray(self):
log("Testing: np.ndarray RHS ...")
self.s.w = np.ndarray((100, 100), dtype=np.uint8)
self.s.w = np.ndarray([2, 2, 2], dtype=np.uint8)
w = np.ndarray([2, 2, 2], dtype=np.uint8)
self.s.w.fill(9)
w.fill(9)
self.assertTrue(np.array_equal(self.s.w, w))
log("Testing: pymm.ndarray RHS ...")
self.s.x = pymm.ndarray((100, 100), dtype=np.uint8)
self.s.x = pymm.ndarray([2, 2, 2], dtype=np.uint8)
x = np.ndarray([2, 2, 2], dtype=np.uint8)
self.s.x.fill(8)
x.fill(8)
self.assertTrue(np.array_equal(self.s.x, x))
self.s.erase('x')
self.s.erase('w')
def test_shelf_transaction(self):
shelf = pymm.shelf('myTransactionsShelf',
size_mb=1024,
pmem_path='/mnt/pmem0',
force_new=True)
shelf.n = pymm.ndarray((100, 100), dtype=np.uint8)
shelf.m = pymm.ndarray((100, 100), dtype=np.uint8)
print(shelf.tx_begin([shelf.m, shelf.n]))
for i in np.arange(0, 10):
shelf.n += 1
shelf.m += 3
print(shelf.items)
shelf.inspect(verbose=False)
shelf.tx_end()
shelf.inspect(verbose=False)
def test_mapstore(self):
self.s.x = pymm.ndarray((
10,
10,
))
self.s.x.fill(33)
self.assertTrue(self.s.x[0][0] == 33)
self.assertTrue(self.s.x[9][9] == 33)
print(self.s.items)
print(self.s.x)
print(self.s.x._value_named_memory.addr())
def test_ndarray(self):
log("Testing: np.ndarray RHS ...")
self.s.w = np.ndarray((100, 100), dtype=np.uint8)
self.s.w = np.ndarray([2, 2, 2], dtype=np.uint8)
w = np.ndarray([2, 2, 2], dtype=np.uint8)
self.s.w.fill(9)
w.fill(9)
if np.array_equal(self.s.w, w) != True:
raise RuntimeError('ndarray equality failed')
log("Testing: pymm.ndarray RHS ...")
self.s.x = pymm.ndarray((100, 100), dtype=np.uint8)
self.s.x = pymm.ndarray([2, 2, 2], dtype=np.uint8)
x = np.ndarray([2, 2, 2], dtype=np.uint8)
self.s.x.fill(8)
x.fill(8)
self.assertTrue(np.array_equal(self.s.x, x))
self.s.erase('x')
self.s.erase('w')
log("Testing: ndarray OK!")
def __init__(self,
num_params: int,
shelf,
dtype: np.dtype = np.float32) -> None:
super().__init__(num_params, dtype=dtype)
self.shelf = shelf
# reserve space on the shelf for the mean and covariance
self.shelf.posterior_sums = pymm.ndarray((num_params, 1),
dtype=self.dtype)
self.shelf.posterior_squaresums = pymm.ndarray(
(num_params, num_params), dtype=self.dtype)
self.shelf.posterior_mu = pymm.ndarray((num_params, 1),
dtype=self.dtype)
self.shelf.posterior_cov = pymm.ndarray((num_params, num_params),
dtype=self.dtype)
# i know that creating an array defaults the values to 0,
# but i want to make sure that this is always the case
self.shelf.posterior_sums.fill(0)
self.shelf.posterior_squaresums.fill(0)
self.shelf.posterior_mu.fill(0)
self.shelf.posterior_cov.fill(0)
def test_check_content(self):
if 'x' in self.s.items:
log('x is there: value={}'.format(self.s.x))
self.s.x += 1.1
else:
log('x is not there!')
self.s.x = 1.0
if 'y' in self.s.items:
log('y is there:')
log(self.s.y)
self.s.y += 1
else:
log('y is not there!')
self.s.y = pymm.ndarray((10,10,),dtype=np.uint32)
self.s.y.fill(0)
print(self.s.items)
def test_write_operations():
'''
Test write operations
'''
import pymm
import numpy as np
# create new shelf (override any existing myShelf)
#
s = pymm.shelf('test_write_operations',32,pmem_path='/mnt/pmem0',force_new=True)
# create variable x on shelf (using shadow type)
s.x = pymm.ndarray((10,10),dtype=np.uint8)
s.x.fill(1)
# do not modify but make copies
-s.x
s.x+8
if s.x[0][0] != 1:
raise RuntimeError('test failed unexpectedly')
return s
def Xtest_transactions(self):
shelf = pymm.shelf('myTransactionsShelf',
size_mb=1024,
pmem_path='/mnt/pmem0',
force_new=True)
log("Testing: transaction on matrix fill ...")
shelf.n = pymm.ndarray((100, 100), dtype=np.uint8)
shelf.n += 1
shelf.n += 1
# shelf.s = 'This is a string!'
# shelf.f = 1.123
# shelf.i = 645338
# shelf.b = b'Hello'
# shelf.w = np.ndarray((100,100),dtype=np.uint8)
# shelf.w.fill(ord('a'))
# shelf.t = pymm.torch_tensor(np.arange(0,10))
print(shelf.items)
shelf.inspect(verbose=False)
shelf.persist()
shelf.inspect(verbose=False)
def test_dram_mapstore(self):
log("Running shelf with mapstore and default MM plugin ...")
s = pymm.shelf('myShelf2',size_mb=8,backend='mapstore') # note, no force_new or pmem_path
s.x = pymm.ndarray((10,10,))
print(s.items)
log("OK!")
