def existing_instance(memory_resource: MemoryResource, name: str):
'''
Determine if an persistent named memory object corresponds to this type
'''
metadata = memory_resource.get_named_memory(name + '-meta')
if metadata is None:
return None
if pymmcore.ndarray_read_header(memoryview(metadata)) == None:
return None
else:
return shelved_ndarray(memory_resource, name, shape = None)
def existing_instance(memory_resource: MemoryResource, name: str):
'''
Determine if an persistent named memory object corresponds to this type
'''
metadata = memory_resource.get_named_memory(name)
if metadata is None:
raise RuntimeError('bad object name')
if pymmcore.ndarray_read_header(memoryview(metadata)) == None:
return (False, None)
else:
return (True, shelved_ndarray(memory_resource, name, shape = None))
def __new__(subtype, memory_resource, name, shape, dtype=float, strides=None, order='C'):
# determine size of memory needed
#
descr = dtypedescr(dtype)
_dbytes = descr.itemsize
if not shape is None:
if not isinstance(shape, tuple):
shape = (shape,)
size = np.intp(1) # avoid default choice of np.int_, which might overflow
for k in shape:
size *= k
value_named_memory = memory_resource.open_named_memory(name)
metadata_key = name + '-meta'
if value_named_memory == None: # does not exist yet
#
# create a newly allocated named memory from MemoryResource
#
value_named_memory = memory_resource.create_named_memory(name,
int(size*_dbytes),
8, # alignment
False) # zero
# construct array using supplied memory
# shape, dtype=float, buffer=None, offset=0, strides=None, order=None
self = np.ndarray.__new__(subtype, dtype=dtype, shape=shape, buffer=value_named_memory.buffer,
strides=strides, order=order)
# create and store metadata header
metadata = pymmcore.ndarray_header(self,np.dtype(dtype).str)
memory_resource.put_named_memory(metadata_key, metadata)
else:
# entity already exists, load metadata
metadata = memory_resource.get_named_memory(metadata_key)
hdr = pymmcore.ndarray_read_header(memoryview(metadata))
self = np.ndarray.__new__(subtype, dtype=hdr['dtype'], shape=hdr['shape'], buffer=value_named_memory.buffer,
strides=hdr['strides'], order=order)
self._memory_resource = memory_resource
self._value_named_memory = value_named_memory
self._metadata_key = metadata_key
self.name = name
return self
def __new__(subtype,
memory_resource,
name,
shape,
dtype=float,
strides=None,
order='C',
requires_grad=False):
#print('shelved_torch_tensor: shape={} dtype={}'.format(shape, dtype))
torch_to_numpy_dtype_dict = {
torch.bool: bool,
torch.uint8: np.uint8,
torch.int8: np.int8,
torch.int16: np.int16,
torch.int32: np.int32,
torch.int64: np.int64,
torch.float16: np.float16,
torch.float32: np.float32,
torch.float64: np.float64,
torch.complex64: np.complex64,
torch.complex128: np.complex128,
}
np_dtype = torch_to_numpy_dtype_dict.get(dtype, None)
value_key = name + '-value'
metadata_key = name
metadata_memory = memory_resource.open_named_memory(metadata_key)
value_memory = memory_resource.open_named_memory(value_key)
if value_memory == None: # does not exist yet
if isinstance(shape, torch.Size):
ndshape = [shape.numel()]
else:
ndshape = np.shape(shape)
# use shelved_ndarray under the hood and make a subclass from it
base_ndarray = shelved_ndarray(memory_resource,
name=name,
shape=ndshape,
dtype=np_dtype,
type=1)
# copy data
if isinstance(shape, list):
base_ndarray[:] = shape
# create and store metadata header : type=1 indicates torch_tensor
metadata = pymmcore.ndarray_header(base_ndarray,
np.dtype(np_dtype).str,
type=1)
memory_resource.put_named_memory(metadata_key, metadata)
else:
# entity already exists, load metadata
hdr = pymmcore.ndarray_read_header(
memoryview(metadata_memory.buffer),
type=1) # type=1 indicate torch_tensor
del value_memory # need to release references
del metadata_memory # need to release references
base_ndarray = shelved_ndarray(memory_resource,
name=name,
dtype=hdr['dtype'],
shape=hdr['shape'],
strides=hdr['strides'],
order=order,
type=1)
self = torch.Tensor._make_subclass(subtype,
torch.as_tensor(base_ndarray))
self._base_ndarray = base_ndarray
self._metadata_named_memory = base_ndarray._metadata_named_memory
self._value_named_memory = base_ndarray._value_named_memory
# hold a reference to the memory resource
self._memory_resource = memory_resource
self._metadata_key = metadata_key
self.name_on_shelf = name
self.requires_grad = requires_grad # by default .grad is not there
self.retains_grad = False # by default .grad is not there
return self
def __new__(subtype, memory_resource, name, shape, dtype=float, strides=None, order='C', type=0, zero=False):
#
# determine size of memory needed
#
descr = dtypedescr(dtype)
_dbytes = descr.itemsize
if shape != None:
size = np.intp(1) # avoid default choice of np.int_, which might overflow
if isinstance(shape, tuple) or isinstance(shape, list):
# if isinstance(shape, tuple):
# if isinstance(shape[0], list):
# shape = shape[0]
for k in shape:
size *= k
else:
raise RuntimeError('unhandled condition in shelved_ndarray shape handling (shape={})'.format(shape))
# the meta data is always accessible by the plain key name
# the value if not concated after meta data is in a separate key with -value suffix
value_key = name + '-value'
metadata_key = name
metadata_memory = memory_resource.open_named_memory(metadata_key)
value_memory = memory_resource.open_named_memory(value_key)
if value_memory == None: # does not exist yet
#
# create a newly allocated named memory from MemoryResource
#
msize = int(size*_dbytes)
if msize < 8:
alignment = 1
else:
alignment = 8
value_memory = memory_resource.create_named_memory(value_key,
msize,
alignment,
zero) # zero memory
assert value_memory != None
# construct array using supplied memory
# shape, dtype=float, buffer=None, offset=0, strides=None, order=None
self = np.ndarray.__new__(subtype, dtype=dtype, shape=shape, buffer=value_memory.buffer,
strides=strides, order=order)
# create and store metadata header
metadata = pymmcore.ndarray_header(self,np.dtype(dtype).str, type=type)
memory_resource.put_named_memory(metadata_key, metadata)
metadata_memory = memory_resource.open_named_memory(metadata_key)
else:
# entity already exists, load metadata
assert metadata_memory != None
hdr = pymmcore.ndarray_read_header(memoryview(metadata_memory.buffer),type=type)
self = np.ndarray.__new__(subtype, dtype=hdr['dtype'], shape=hdr['shape'], buffer=value_memory.buffer,
strides=hdr['strides'], order=order)
assert value_memory != None
assert metadata_memory != None
# hold a reference to the memory resource
self._memory_resource = memory_resource
self._value_named_memory = value_memory
self._metadata_named_memory = metadata_memory
self._metadata_key = metadata_key
self._value_key = value_key
self.name = name
self._use_wbinvd = os.path.isfile('/proc/wbinvd')
return self