def new_stream(self,
mimetype=params.MediaType('application', 'octet-stream'),
created=None):
"""Creates a new stream in the store.
mimetype
A :py:class:`~pyslet.http.params.MediaType` object
Returns a stream entity which is an
:py:class:`~pyslet.odata2.csdl.Entity` instance.
The stream is identified by the stream entity's key which you
can store elsewhere as a reference and pass to
:py:meth:`get_stream` to retrieve the stream again later."""
with self.stream_set.OpenCollection() as streams:
stream = streams.new_entity()
if not isinstance(mimetype, params.MediaType):
mimetype = params.MediaType.from_str(mimetype)
stream['mimetype'].set_from_value(str(mimetype))
now = TimePoint.from_now_utc()
stream['size'].set_from_value(0)
if created is None:
stream['created'].set_from_value(now)
stream['modified'].set_from_value(now)
else:
created = created.shift_zone(0)
stream['created'].set_from_value(created)
stream['modified'].set_from_value(created)
stream['md5'].set_from_value(hashlib.md5().digest())
streams.insert_entity(stream)
return stream
def flush(self):
if self._bdirty:
# the current block is dirty, write it out
data = self._bdata[:self._btop]
if data:
block = self.blocks[self._bnum]
if block.exists:
self.ss.update_block(block, str(data))
else:
self.blocks[self._bnum] = self.ss.store_block(
self.stream, self._bnum, data)
if self._md5 is not None and self._bnum == self._md5num:
self._md5.update(str(data))
self._md5num += 1
else:
self._md5 = None
if self.size != self.stream['size'].value:
self.stream['size'].set_from_value(self.size)
now = TimePoint.from_now_utc()
self.stream['modified'].set_from_value(now)
if self._md5 is not None:
self.stream['md5'].set_from_value(self._md5.digest())
else:
self.stream['md5'].set_null()
self.stream.commit()
self._bdirty = False
def flush(self):
if self._bdirty:
# the current block is dirty, write it out
data = self._bdata[:self._btop]
if data:
block = self.blocks[self._bnum]
if block.exists:
self.ss.update_block(block, str(data))
else:
self.blocks[self._bnum] = self.ss.store_block(
self.stream, self._bnum, data)
if self._md5 is not None and self._bnum == self._md5num:
self._md5.update(str(data))
self._md5num += 1
else:
self._md5 = None
if self.size != self.stream['size'].value:
self.stream['size'].set_from_value(self.size)
now = TimePoint.from_now_utc()
self.stream['modified'].set_from_value(now)
if self._md5 is not None:
self.stream['md5'].set_from_value(self._md5.digest())
else:
self.stream['md5'].set_null()
self.stream.commit()
self._bdirty = False
def new_stream(self,
mimetype=params.MediaType('application', 'octet-stream'),
created=None):
"""Creates a new stream in the store.
mimetype
A :py:class:`~pyslet.http.params.MediaType` object
Returns a stream entity which is an
:py:class:`~pyslet.odata2.csdl.Entity` instance.
The stream is identified by the stream entity's key which you
can store elsewhere as a reference and pass to
:py:meth:`get_stream` to retrieve the stream again later."""
with self.stream_set.OpenCollection() as streams:
stream = streams.new_entity()
if not isinstance(mimetype, params.MediaType):
mimetype = params.MediaType.from_str(mimetype)
stream['mimetype'].set_from_value(str(mimetype))
now = TimePoint.from_now_utc()
stream['size'].set_from_value(0)
if created is None:
stream['created'].set_from_value(now)
stream['modified'].set_from_value(now)
else:
created = created.shift_zone(0)
stream['created'].set_from_value(created)
stream['modified'].set_from_value(created)
stream['md5'].set_from_value(hashlib.md5().digest())
streams.insert_entity(stream)
return stream
def lock(self, hash_key, timeout=60):
"""Acquires the lock on hash_key or raises LockError
The return value is a context manager object that will
automatically release the lock on hash_key when it exits.
locks are not nestable, they can only be acquired once. If the
lock cannot be acquired a back-off strategy is implemented using
random waits up to a total maximum of *timeout* seconds. If the
lock still cannot be obtained :py:class:`LockError` is raised."""
owner = "%s_%i" % (self.magic, threading.current_thread().ident)
with self.entity_set.OpenCollection() as locks:
tnow = time.time()
tstop = tnow + timeout
twait = 0
while tnow < tstop:
time.sleep(twait)
lock = locks.new_entity()
lock['hash'].set_from_value(hash_key)
lock['owner'].set_from_value(owner)
lock['created'].set_from_value(TimePoint.from_now_utc())
try:
locks.insert_entity(lock)
return LockStoreContext(self, hash_key)
except edm.ConstraintError:
pass
try:
lock = locks[hash_key]
except KeyError:
# someone deleted the lock, go straight round again
twait = 0
tnow = time.time()
continue
# has this lock expired?
locktime = lock['created'].value.with_zone(zdirection=0)
if locktime.get_unixtime() + self.lock_timeout < tnow:
# use optimistic locking
lock['owner'].set_from_value(owner)
try:
locks.update_entity(lock)
logging.warn(
"LockingBlockStore removed stale lock "
"on %s", hash_key)
return LockStoreContext(self, hash_key)
except KeyError:
twait = 0
tnow = time.time()
continue
except edm.ConstraintError:
pass
twait = random.randint(0, timeout // 5)
tnow = time.time()
logging.warn("LockingBlockStore: timeout locking %s", hash_key)
raise LockError
def test_store(self):
ss = blockstore.StreamStore(bs=self.bs,
ls=self.ls,
entity_set=self.cdef['Streams'])
with self.cdef['Streams'].open() as streams:
with self.cdef['BlockLists'].open() as blocks:
stream1 = streams.new_entity()
stream1['mimetype'].set_from_value("text/plain")
now = TimePoint.from_now_utc()
stream1['created'].set_from_value(now)
stream1['modified'].set_from_value(now)
streams.insert_entity(stream1)
stream2 = streams.new_entity()
stream2['mimetype'].set_from_value("text/plain")
now = TimePoint.from_now_utc()
stream2['created'].set_from_value(now)
stream2['modified'].set_from_value(now)
streams.insert_entity(stream2)
fox = b"The quick brown fox jumped over the lazy dog"
cafe = ul("Caf\xe9").encode('utf-8')
ss.store_block(stream1, 0, cafe)
ss.store_block(stream1, 1, fox)
ss.store_block(stream2, 0, cafe)
self.assertTrue(len(blocks) == 3)
blocks1 = list(ss.retrieve_blocklist(stream1))
self.assertTrue(ss.retrieve_block(blocks1[0]) == cafe)
self.assertTrue(ss.retrieve_block(blocks1[1]) == fox)
blocks2 = list(ss.retrieve_blocklist(stream2))
self.assertTrue(ss.retrieve_block(blocks2[0]) == cafe)
ss.delete_blocks(stream1, 1)
# should also have deleted fox from the block store
self.assertTrue(len(blocks) == 2)
try:
ss.retrieve_block(blocks1[1])
self.fail("Expected missing block")
except blockstore.BlockMissing:
pass
self.assertTrue(ss.retrieve_block(blocks1[0]) == cafe)
ss.delete_blocks(stream1)
self.assertTrue(len(blocks) == 1)
self.assertTrue(ss.retrieve_block(blocks2[0]) == cafe)
def lock(self, hash_key, timeout=60):
"""Acquires the lock on hash_key or raises LockError
The return value is a context manager object that will
automatically release the lock on hash_key when it exits.
locks are not nestable, they can only be acquired once. If the
lock cannot be acquired a back-off strategy is implemented using
random waits up to a total maximum of *timeout* seconds. If the
lock still cannot be obtained :py:class:`LockError` is raised."""
owner = "%s_%i" % (self.magic, threading.current_thread().ident)
with self.entity_set.OpenCollection() as locks:
tnow = time.time()
tstop = tnow + timeout
twait = 0
while tnow < tstop:
time.sleep(twait)
lock = locks.new_entity()
lock['hash'].set_from_value(hash_key)
lock['owner'].set_from_value(owner)
lock['created'].set_from_value(TimePoint.from_now_utc())
try:
locks.insert_entity(lock)
return LockStoreContext(self, hash_key)
except edm.ConstraintError:
pass
try:
lock = locks[hash_key]
except KeyError:
# someone deleted the lock, go straight round again
twait = 0
tnow = time.time()
continue
# has this lock expired?
locktime = lock['created'].value.with_zone(zdirection=0)
if locktime.get_unixtime() + self.lock_timeout < tnow:
# use optimistic locking
lock['owner'].set_from_value(owner)
try:
locks.update_entity(lock)
logging.warn("LockingBlockStore removed stale lock "
"on %s", hash_key)
return LockStoreContext(self, hash_key)
except KeyError:
twait = 0
tnow = time.time()
continue
except edm.ConstraintError:
pass
twait = random.randint(0, timeout // 5)
tnow = time.time()
logging.warn("LockingBlockStore: timeout locking %s", hash_key)
raise LockError
def test_store(self):
ss = blockstore.StreamStore(bs=self.bs, ls=self.ls,
entity_set=self.cdef['Streams'])
with self.cdef['Streams'].open() as streams:
with self.cdef['BlockLists'].open() as blocks:
stream1 = streams.new_entity()
stream1['mimetype'].set_from_value("text/plain")
now = TimePoint.from_now_utc()
stream1['created'].set_from_value(now)
stream1['modified'].set_from_value(now)
streams.insert_entity(stream1)
stream2 = streams.new_entity()
stream2['mimetype'].set_from_value("text/plain")
now = TimePoint.from_now_utc()
stream2['created'].set_from_value(now)
stream2['modified'].set_from_value(now)
streams.insert_entity(stream2)
fox = b"The quick brown fox jumped over the lazy dog"
cafe = ul("Caf\xe9").encode('utf-8')
ss.store_block(stream1, 0, cafe)
ss.store_block(stream1, 1, fox)
ss.store_block(stream2, 0, cafe)
self.assertTrue(len(blocks) == 3)
blocks1 = list(ss.retrieve_blocklist(stream1))
self.assertTrue(ss.retrieve_block(blocks1[0]) == cafe)
self.assertTrue(ss.retrieve_block(blocks1[1]) == fox)
blocks2 = list(ss.retrieve_blocklist(stream2))
self.assertTrue(ss.retrieve_block(blocks2[0]) == cafe)
ss.delete_blocks(stream1, 1)
# should also have deleted fox from the block store
self.assertTrue(len(blocks) == 2)
try:
ss.retrieve_block(blocks1[1])
self.fail("Expected missing block")
except blockstore.BlockMissing:
pass
self.assertTrue(ss.retrieve_block(blocks1[0]) == cafe)
ss.delete_blocks(stream1)
self.assertTrue(len(blocks) == 1)
self.assertTrue(ss.retrieve_block(blocks2[0]) == cafe)
def flush(self):
if self._bdirty:
# the current block is dirty, write it out
data = self._bdata[:self._btop]
if data:
block = self.blocks[self._bnum]
if block.exists:
self.ss.update_block(block, str(data))
else:
self.blocks[self._bnum] = self.ss.store_block(
self.stream, self._bnum, data)
if self.size != self.stream['size'].value:
self.stream['size'].SetFromValue(self.size)
now = TimePoint.FromNowUTC()
self.stream['modified'].SetFromValue(now)
self.stream.Update()
self._bdirty = False
def new_stream(self,
mimetype=http.MediaType('application', 'octet-stream')):
"""Creates a new stream in the store.
mimetype
A :py:class:`~pyslet.rfc2616.MimeType` object
Returns a stream entity which is an
:py:class:`~pyslet.odata2.csdl.Entity` instance.
The stream is identified by the stream entity's key which
you can store elsewhere as a reference."""
with self.stream_set.OpenCollection() as streams:
stream = streams.new_entity()
if not isinstance(mimetype, http.MediaType):
mimetype = http.MediaType.FromString(mimetype)
stream['mimetype'].SetFromValue(str(mimetype))
now = TimePoint.FromNowUTC()
stream['size'].SetFromValue(0)
stream['created'].SetFromValue(now)
stream['modified'].SetFromValue(now)
streams.insert_entity(stream)
return stream
