def test_open_w_r(self):
ss = blockstore.StreamStore(bs=self.bs,
ls=self.ls,
entity_set=self.cdef['Streams'])
s1 = ss.new_stream("text/plain")
self.assertTrue(s1['md5'].value == hashlib.md5().digest())
with ss.open_stream(s1, 'w') as s:
self.assertFalse(s.closed)
self.assertFalse(s.readable())
self.assertTrue(s.writable())
self.assertTrue(s.tell() == 0)
# try writing a multi-block string
nbytes = 0
fox = b"The quick brown fox jumped over the lazy dog"
cafe = ul("Caf\xe9").encode('utf-8')
data = fox + cafe + fox
while nbytes < len(data):
nbytes += s.write(data[nbytes:])
self.assertTrue(s.tell() == nbytes)
self.assertTrue(s1['size'].value == nbytes)
self.assertTrue(s1['md5'].value == hashlib.md5(data).digest())
with self.cdef['BlockLists'].open() as blocks:
# data should spill over to 2 blocks
self.assertTrue(len(blocks) == 2)
with ss.open_stream(s1, 'r') as s:
self.assertFalse(s.closed)
self.assertTrue(s.readable())
self.assertFalse(s.writable())
self.assertTrue(s.tell() == 0)
rdata = s.read()
self.assertTrue(rdata == data, "Read back %s" % repr(rdata))
self.assertTrue(s.tell() == nbytes)
def test_create(self):
ss = blockstore.StreamStore(bs=self.bs,
ls=self.ls,
entity_set=self.cdef['Streams'])
s1 = ss.new_stream("text/plain")
self.assertTrue(isinstance(s1, edm.Entity))
self.assertTrue(s1.exists)
try:
self.assertTrue(s1.key() is not None)
except KeyError:
self.fail("stream entity has not key")
self.assertTrue(s1['mimetype'].value == "text/plain")
s2 = ss.new_stream(
params.MediaType('text', 'plain',
{'charset': ('charset', 'utf-8')}))
self.assertTrue(isinstance(s2, edm.Entity))
self.assertTrue(s2['mimetype'].value == "text/plain; charset=utf-8")
skey1 = s1.key()
skey2 = s2.key()
s1 = ss.get_stream(skey1)
self.assertTrue(isinstance(s1, edm.Entity))
for i in range3(10):
try:
ss.get_stream(i)
self.assertTrue(i == skey1 or i == skey2)
except KeyError:
self.assertFalse(i == skey1 or i == skey2)
def test_mem_mem(self):
self.container = InMemoryEntityContainer(self.cdef)
self.bs = blockstore.EDMBlockStore(entity_set=self.cdef['Blocks'],
max_block_size=self.block_size)
self.ls = blockstore.LockStore(entity_set=self.cdef['BlockLocks'])
self.ss = blockstore.StreamStore(bs=self.bs,
ls=self.ls,
entity_set=self.cdef['Streams'])
self.random_rw()
def test_sql_sql(self):
self.container = BlockStoreContainer(container=self.cdef,
file_path=str(
self.d.join('blockstore.db')))
self.container.create_all_tables()
self.bs = blockstore.EDMBlockStore(entity_set=self.cdef['Blocks'],
max_block_size=self.block_size)
self.ls = blockstore.LockStore(entity_set=self.cdef['BlockLocks'])
self.ss = blockstore.StreamStore(bs=self.bs,
ls=self.ls,
entity_set=self.cdef['Streams'])
self.random_rw()
def test_store(self):
ss = blockstore.StreamStore(bs=self.bs,
ls=self.ls,
entity_set=self.cdef['Streams'])
with self.cdef['Streams'].open() as streams:
with self.cdef['BlockLists'].open() as blocks:
stream1 = streams.new_entity()
stream1['mimetype'].set_from_value("text/plain")
now = TimePoint.from_now_utc()
stream1['created'].set_from_value(now)
stream1['modified'].set_from_value(now)
streams.insert_entity(stream1)
stream2 = streams.new_entity()
stream2['mimetype'].set_from_value("text/plain")
now = TimePoint.from_now_utc()
stream2['created'].set_from_value(now)
stream2['modified'].set_from_value(now)
streams.insert_entity(stream2)
fox = b"The quick brown fox jumped over the lazy dog"
cafe = ul("Caf\xe9").encode('utf-8')
ss.store_block(stream1, 0, cafe)
ss.store_block(stream1, 1, fox)
ss.store_block(stream2, 0, cafe)
self.assertTrue(len(blocks) == 3)
blocks1 = list(ss.retrieve_blocklist(stream1))
self.assertTrue(ss.retrieve_block(blocks1[0]) == cafe)
self.assertTrue(ss.retrieve_block(blocks1[1]) == fox)
blocks2 = list(ss.retrieve_blocklist(stream2))
self.assertTrue(ss.retrieve_block(blocks2[0]) == cafe)
ss.delete_blocks(stream1, 1)
# should also have deleted fox from the block store
self.assertTrue(len(blocks) == 2)
try:
ss.retrieve_block(blocks1[1])
self.fail("Expected missing block")
except blockstore.BlockMissing:
pass
self.assertTrue(ss.retrieve_block(blocks1[0]) == cafe)
ss.delete_blocks(stream1)
self.assertTrue(len(blocks) == 1)
self.assertTrue(ss.retrieve_block(blocks2[0]) == cafe)
def test_open_default(self):
ss = blockstore.StreamStore(bs=self.bs,
ls=self.ls,
entity_set=self.cdef['Streams'])
s1 = ss.new_stream("text/plain")
self.assertTrue(s1['size'].value == 0)
with ss.open_stream(s1) as s:
self.assertTrue(isinstance(s, io.RawIOBase))
self.assertFalse(s.closed)
try:
s.fileno()
self.fail("streams do not have file numbers")
except IOError:
pass
self.assertFalse(s.isatty())
self.assertTrue(s.readable())
self.assertFalse(s.writable())
self.assertTrue(s.tell() == 0)
self.assertTrue(s.seekable())
# the stream is empty, so read should return EOF
self.assertTrue(len(s.read()) == 0)
def test_init(self):
blockstore.StreamStore(bs=self.bs,
ls=self.ls,
entity_set=self.cdef['Streams'])