def test_large_hash(self):
with open('/dev/urandom', 'r+b') as f:
buf = StringBuffer.from_file(f.fileno(), 4096 * 3)
buf_sha = buf.hash()
from hashlib import sha1
outside = StringBuffer(sha1(buf.raw_read()).digest())
self.assertEquals(outside.raw_read(), buf_sha.raw_read())
def create_symbol_tables(self):
st = SymbolTableHeader()
st.add('root')
st.add('time')
st.add('time.long_name:universal time')
st.add('time.reference:UTC')
st.add('time.units:minutes since 2000-01-01 00:00')
st.add('ncells')
st.add('ncells.long_name:sequential cell count')
st.add('lon')
st.add('lon.long_name:Cell longitude')
st.add('lon.units:degrees_east')
st.add('lat')
st.add('lat.long_name:Cell latitude')
st.add('lat.units:degrees_north')
st.add('U')
st.add('U.long_name:eastward water velocity')
st.add('V')
st.add('V.long_name:northward water velocity')
with TemporaryFile('w+b') as f:
st.serialize().fwrite(f.fileno())
f.seek(0)
sb = StringBuffer.from_file(f.fileno(),16)
header = MFSObjectHeader.deserialize(sb)
symbol_buf = StringBuffer.from_file(f.fileno(), header.total_size)
header.deserialize_table(symbol_buf)
i = random.randint(0, len(st.symbols)-1)
self.assertEquals(st.symbols[i].symbol, header.symbols[i].symbol)
def create_symbol_tables(self):
st = SymbolTableHeader()
st.add('root')
st.add('time')
st.add('time.long_name:universal time')
st.add('time.reference:UTC')
st.add('time.units:minutes since 2000-01-01 00:00')
st.add('ncells')
st.add('ncells.long_name:sequential cell count')
st.add('lon')
st.add('lon.long_name:Cell longitude')
st.add('lon.units:degrees_east')
st.add('lat')
st.add('lat.long_name:Cell latitude')
st.add('lat.units:degrees_north')
st.add('U')
st.add('U.long_name:eastward water velocity')
st.add('V')
st.add('V.long_name:northward water velocity')
with TemporaryFile('w+b') as f:
st.serialize().fwrite(f.fileno())
f.seek(0)
sb = StringBuffer.from_file(f.fileno(), 16)
header = MFSObjectHeader.deserialize(sb)
symbol_buf = StringBuffer.from_file(f.fileno(), header.total_size)
header.deserialize_table(symbol_buf)
i = random.randint(0, len(st.symbols) - 1)
self.assertEquals(st.symbols[i].symbol, header.symbols[i].symbol)
def serialize(self):
sb = StringBuffer(16 + self.total_size)
sb.pack('<BBH', self.mfs_type, self.version, self.children)
sb.write(self.signature)
sb.pack('<Q', self.total_size)
for o in self.objects:
sb.write(o.serialize())
return sb
def create_datatype(self, dtype_class):
with TemporaryFile('w+b') as f:
dtype_h = dtype_class()
sb = dtype_h.serialize()
sb.fwrite(f.fileno())
f.seek(0)
sb = StringBuffer(16)
sb.fread(f.fileno())
sb.seek(0)
dtype_h = MFSObjectHeader.deserialize(sb)
self.assertIsInstance(dtype_h, DatatypeHeader)
def serialize(self):
sb = StringBuffer(24)
sb.pack('<BBH', self.mfs_type, self.mode, self.flags)
sb.write(self.sha)
sb.seek(0)
return sb
def serialize(self):
sb = StringBuffer(16 + self.total_size)
sb.pack('<BBHIQ', self.mfs_type, 0, 0, self.entry_no, self.total_size)
for symbol in self.symbols:
sb.write(symbol.serialize())
sb.seek(0)
return sb
def serialize(self):
sb = StringBuffer(16 + self.total_size)
sb.pack('<BBHHHQ', self.mfs_type, self.ver, self.name_size,
self.datatype_size, self.dataspace_size, self.total_size)
if self.attr is not None:
sb.write(self.attr.serialize())
sb.seek(0)
return sb
def serialize(self):
sb = StringBuffer(16 + self.total_size)
sb.pack('<BBBBIQ', self.mfs_type, self.ver, self.dims, self.flags, 0,
self.total_size)
for ds in self.dataspaces:
sb.write(ds.serialize())
sb.seek(0)
return sb
def serialize(self):
sb = StringBuffer(8 + self.symbol_length)
sb.pack('<HHI', self.idx, 0,
max(len(self.symbol) + 1,
self.symbol_length)) # Account for null-terminator
sb.write(self.symbol)
sb.seek(0)
return sb
def create_dataspace(self):
ds = DataspaceHeader((40,40,40))
sb = ds.serialize()
with TemporaryFile('w+b') as f:
sb.fwrite(f.fileno())
f.seek(0)
sb = StringBuffer.from_file(f.fileno(), 16)
ds_header = MFSObjectHeader.deserialize(sb)
sb = StringBuffer.from_file(f.fileno(), ds_header.total_size)
ds_header.deserialize_dataspaces(sb)
for i in xrange(3):
self.assertEquals(ds_header.dataspaces[i].dim_size, 40)
def header_check(self, f, dtype_h, datatype, flags, size, total_size):
sb = dtype_h.serialize()
sb.fwrite(f.fileno())
f.seek(0)
sb = StringBuffer(16)
sb.fread(f.fileno())
sb.seek(0)
dtype_h = MFSObjectHeader.deserialize(sb)
self.assertIsInstance(dtype_h, DatatypeHeader)
self.assertEquals(dtype_h.datatype, datatype)
self.assertEquals(dtype_h.flags, flags)
self.assertEquals(dtype_h.size, size)
self.assertEquals(dtype_h.total_size, total_size)
def create_dataspace(self):
ds = DataspaceHeader((40, 40, 40))
sb = ds.serialize()
with TemporaryFile('w+b') as f:
sb.fwrite(f.fileno())
f.seek(0)
sb = StringBuffer.from_file(f.fileno(), 16)
ds_header = MFSObjectHeader.deserialize(sb)
sb = StringBuffer.from_file(f.fileno(), ds_header.total_size)
ds_header.deserialize_dataspaces(sb)
for i in xrange(3):
self.assertEquals(ds_header.dataspaces[i].dim_size, 40)
def test_from_file(self):
with TemporaryFile('w+b') as f:
b = 'hello world'
f.write(b)
f.seek(0)
sb = StringBuffer.from_file(f.fileno(), len(b)+1)
self.assertEquals(sb.read(), b)
def serialize(self):
sb = StringBuffer(16 + self.total_size)
sb.pack('<BBBBIQ', self.mfs_type, self.ver, self.dims, self.flags, 0, self.total_size)
for ds in self.dataspaces:
sb.write(ds.serialize())
sb.seek(0)
return sb
def serialize(self):
sb = StringBuffer(16 + self.total_size)
sb.pack('<BBHIQ', self.mfs_type, 0, 0, self.entry_no, self.total_size)
for symbol in self.symbols:
sb.write(symbol.serialize())
sb.seek(0)
return sb
def serialize(self):
sb = StringBuffer(16 + self.total_size)
sb.pack('<BBHHHQ', self.mfs_type, self.ver, self.name_size, self.datatype_size, self.dataspace_size, self.total_size)
if self.attr is not None:
sb.write(self.attr.serialize())
sb.seek(0)
return sb
def test_fread(self):
sb = StringBuffer(16)
with TemporaryFile('w+b') as f:
fd = f.fileno()
f.write('hello world\n')
f.seek(0)
sb.fread(fd, 8)
sb.seek(0)
self.assertEquals(sb.raw_read(8), 'hello wo')
def test_dataspace_header(self):
with TemporaryFile('w+b') as f:
dataspace = DataspaceHeader((2, 10))
sb = dataspace.serialize()
sb.fwrite(f.fileno())
f.seek(0)
sb = StringBuffer.from_file(f.fileno(), 16)
dataspace = MFSObjectHeader.deserialize(sb)
self.assertIsInstance(dataspace, DataspaceHeader)
self.assertEquals(dataspace.ver, 0)
self.assertEquals(dataspace.dims, 2)
self.assertEquals(dataspace.flags, 0)
self.assertEquals(dataspace.total_size, 16)
sb = StringBuffer.from_file(f.fileno(), dataspace.total_size)
dataspace.deserialize_dataspaces(sb)
self.assertEquals(dataspace.dataspaces[0].dim_size, 2)
self.assertEquals(dataspace.dataspaces[1].dim_size, 10)
def test_dataspace_header(self):
with TemporaryFile('w+b') as f:
dataspace = DataspaceHeader((2,10))
sb = dataspace.serialize()
sb.fwrite(f.fileno())
f.seek(0)
sb = StringBuffer.from_file(f.fileno(), 16)
dataspace = MFSObjectHeader.deserialize(sb)
self.assertIsInstance(dataspace,DataspaceHeader)
self.assertEquals(dataspace.ver, 0)
self.assertEquals(dataspace.dims, 2)
self.assertEquals(dataspace.flags, 0)
self.assertEquals(dataspace.total_size, 16)
sb = StringBuffer.from_file(f.fileno(), dataspace.total_size)
dataspace.deserialize_dataspaces(sb)
self.assertEquals(dataspace.dataspaces[0].dim_size, 2)
self.assertEquals(dataspace.dataspaces[1].dim_size, 10)
def test_merkle_node(self):
buf = 'sample text'
sha_bytes = sha1(buf).digest()
o = MerkleNode(3, 0, 0, sha_bytes)
mnode = MerkleNodeHeader()
for i in xrange(5):
mnode.add_child(o)
with TemporaryFile('w+b') as f:
mnode.serialize().fwrite(f.fileno())
f.seek(0)
sb = StringBuffer.from_file(f.fileno(), 16)
mnode_header = MFSObjectHeader.deserialize(sb)
sb = StringBuffer.from_file(f.fileno(), mnode_header.total_size)
mnode_header.deserialize_children(sb)
self.assertEquals(mnode.objects[-1].sha, mnode_header.objects[-1].sha)
def test_merkle_node(self):
buf = 'sample text'
sha_bytes = sha1(buf).digest()
o = MerkleNode(3, 0, 0, sha_bytes)
mnode = MerkleNodeHeader()
for i in xrange(5):
mnode.add_child(o)
with TemporaryFile('w+b') as f:
mnode.serialize().fwrite(f.fileno())
f.seek(0)
sb = StringBuffer.from_file(f.fileno(), 16)
mnode_header = MFSObjectHeader.deserialize(sb)
sb = StringBuffer.from_file(f.fileno(), mnode_header.total_size)
mnode_header.deserialize_children(sb)
self.assertEquals(mnode.objects[-1].sha, mnode_header.objects[-1].sha)
def test_fwrite(self):
sb = StringBuffer(24)
sb.set('hello world\n')
with TemporaryFile('w+b') as f:
fd = f.fileno()
sb.fwrite(fd)
f.seek(0)
buf = f.readline()
self.assertEquals(buf, 'hello world\n')
def test_symbol_table_write(self):
st = SymbolTableHeader()
st.add('root')
st.add('') # Null symbol
st.add('time')
st.add('time.units:seconds since 1900-01-01')
st.add('lat')
st.add('lat.units:degrees_north')
st.add('lon.units:degrees_east')
with TemporaryFile('w+b') as f:
sb = st.serialize()
sb.fwrite(f.fileno())
f.seek(0)
sb = StringBuffer.from_file(f.fileno(), 16)
header = MFSObjectHeader.deserialize(sb)
symbol_buf = StringBuffer.from_file(f.fileno(), header.total_size)
header.deserialize_table(symbol_buf)
for i in xrange(len(st.symbols)):
self.assertEquals(st.symbols[i].symbol, header.symbols[i].symbol)
def test_symbol_table_write(self):
st = SymbolTableHeader()
st.add('root')
st.add('') # Null symbol
st.add('time')
st.add('time.units:seconds since 1900-01-01')
st.add('lat')
st.add('lat.units:degrees_north')
st.add('lon.units:degrees_east')
with TemporaryFile('w+b') as f:
sb = st.serialize()
sb.fwrite(f.fileno())
f.seek(0)
sb = StringBuffer.from_file(f.fileno(), 16)
header = MFSObjectHeader.deserialize(sb)
symbol_buf = StringBuffer.from_file(f.fileno(), header.total_size)
header.deserialize_table(symbol_buf)
for i in xrange(len(st.symbols)):
self.assertEquals(st.symbols[i].symbol, header.symbols[i].symbol)
def create_datatype(self, dtype_class):
with TemporaryFile('w+b') as f:
dtype_h = dtype_class()
sb = dtype_h.serialize()
sb.fwrite(f.fileno())
f.seek(0)
sb = StringBuffer(16)
sb.fread(f.fileno())
sb.seek(0)
dtype_h = MFSObjectHeader.deserialize(sb)
self.assertIsInstance(dtype_h, DatatypeHeader)
def header_check(self, f, dtype_h, datatype, flags, size, total_size):
sb = dtype_h.serialize()
sb.fwrite(f.fileno())
f.seek(0)
sb = StringBuffer(16)
sb.fread(f.fileno())
sb.seek(0)
dtype_h = MFSObjectHeader.deserialize(sb)
self.assertIsInstance(dtype_h, DatatypeHeader)
self.assertEquals(dtype_h.datatype, datatype)
self.assertEquals(dtype_h.flags, flags)
self.assertEquals(dtype_h.size, size)
self.assertEquals(dtype_h.total_size, total_size)
def test_buffer(self):
sb = StringBuffer(128)
sb.write('hello world')
sb.seek(0)
self.assertEquals(sb.read(), 'hello world')
self.assertEquals(sb.offset(), 0)
sb.write('jello')
self.assertEquals(sb.offset(), 5)
sb.seek(0)
self.assertEquals(sb.read(), 'jello world')
sb.set('hi')
self.assertEquals(sb.offset(), 0)
self.assertEquals(sb.read(), 'hi')
sb = StringBuffer("Hello World")
self.assertEquals(sb.read(), 'Hello World')
sb = StringBuffer(sb)
self.assertEquals(sb.read(), 'Hello World')
def serialize(self):
sb = StringBuffer(8 + self.symbol_length)
sb.pack('<HHI', self.idx, 0, max(len(self.symbol)+1, self.symbol_length)) # Account for null-terminator
sb.write(self.symbol)
sb.seek(0)
return sb
def serialize(self):
sb = StringBuffer(8)
sb.pack('<Q', self.dim_size)
sb.seek(0)
return sb
def serialize(self):
sb = StringBuffer(16)
sb.pack('<BBHIQ', self.mfs_type, self.datatype, self.flags, self.size, self.total_size)
sb.seek(0)
return sb
def serialize(self):
sb = StringBuffer(8)
sb.pack('<Q', self.dim_size)
sb.seek(0)
return sb
def test_hash(self):
buf = StringBuffer("Hello World")
buf_sha = buf.hash()
from hashlib import sha1
outside = StringBuffer(sha1(buf.raw_read()).digest())
self.assertEquals(outside.raw_read(), buf_sha.raw_read())
def test_overflow(self):
sb = StringBuffer(24)
with self.assertRaises(BufferOverflow):
sb.write('this string is definitely longer than 24 characters and should raise an error')
self.assertRaises(OverflowError, sb.seek, -1)
sb = StringBuffer(8)
sb.set('01234567')
sb.seek(0)
buf = sb.raw_read(8)
self.assertEquals(buf, '01234567')
self.assertRaises(BufferOverflow, sb.raw_read, 1)
def test_word_align(self):
sb = StringBuffer(12)
self.assertEquals(sb.buffer_size(), 16)
def test_read_uint(self):
sb = StringBuffer(16)
sb.pack('<H', 16)
sb.seek(0)
i = sb.read_uint(2)
self.assertEquals(i, 16)
self.assertEquals(sb.offset(), 2)
sb.pack('<Q', 1024)
sb.seek(2)
i = sb.read_uint(8)
self.assertEquals(1024, i)
def test_write_sb(self):
container = StringBuffer(24)
el = StringBuffer(16)
el.set('hello world')
container.write(el)
container.seek(0)
self.assertEquals(container.read(), 'hello world')