def new_stream(self, mimetype=params.MediaType('application', 'octet-stream'), created=None): """Creates a new stream in the store. mimetype A :py:class:`~pyslet.http.params.MediaType` object Returns a stream entity which is an :py:class:`~pyslet.odata2.csdl.Entity` instance. The stream is identified by the stream entity's key which you can store elsewhere as a reference and pass to :py:meth:`get_stream` to retrieve the stream again later.""" with self.stream_set.OpenCollection() as streams: stream = streams.new_entity() if not isinstance(mimetype, params.MediaType): mimetype = params.MediaType.from_str(mimetype) stream['mimetype'].set_from_value(str(mimetype)) now = TimePoint.from_now_utc() stream['size'].set_from_value(0) if created is None: stream['created'].set_from_value(now) stream['modified'].set_from_value(now) else: created = created.shift_zone(0) stream['created'].set_from_value(created) stream['modified'].set_from_value(created) stream['md5'].set_from_value(hashlib.md5().digest()) streams.insert_entity(stream) return stream
def flush(self): if self._bdirty: # the current block is dirty, write it out data = self._bdata[:self._btop] if data: block = self.blocks[self._bnum] if block.exists: self.ss.update_block(block, str(data)) else: self.blocks[self._bnum] = self.ss.store_block( self.stream, self._bnum, data) if self._md5 is not None and self._bnum == self._md5num: self._md5.update(str(data)) self._md5num += 1 else: self._md5 = None if self.size != self.stream['size'].value: self.stream['size'].set_from_value(self.size) now = TimePoint.from_now_utc() self.stream['modified'].set_from_value(now) if self._md5 is not None: self.stream['md5'].set_from_value(self._md5.digest()) else: self.stream['md5'].set_null() self.stream.commit() self._bdirty = False
def lock(self, hash_key, timeout=60): """Acquires the lock on hash_key or raises LockError The return value is a context manager object that will automatically release the lock on hash_key when it exits. locks are not nestable, they can only be acquired once. If the lock cannot be acquired a back-off strategy is implemented using random waits up to a total maximum of *timeout* seconds. If the lock still cannot be obtained :py:class:`LockError` is raised.""" owner = "%s_%i" % (self.magic, threading.current_thread().ident) with self.entity_set.OpenCollection() as locks: tnow = time.time() tstop = tnow + timeout twait = 0 while tnow < tstop: time.sleep(twait) lock = locks.new_entity() lock['hash'].set_from_value(hash_key) lock['owner'].set_from_value(owner) lock['created'].set_from_value(TimePoint.from_now_utc()) try: locks.insert_entity(lock) return LockStoreContext(self, hash_key) except edm.ConstraintError: pass try: lock = locks[hash_key] except KeyError: # someone deleted the lock, go straight round again twait = 0 tnow = time.time() continue # has this lock expired? locktime = lock['created'].value.with_zone(zdirection=0) if locktime.get_unixtime() + self.lock_timeout < tnow: # use optimistic locking lock['owner'].set_from_value(owner) try: locks.update_entity(lock) logging.warn( "LockingBlockStore removed stale lock " "on %s", hash_key) return LockStoreContext(self, hash_key) except KeyError: twait = 0 tnow = time.time() continue except edm.ConstraintError: pass twait = random.randint(0, timeout // 5) tnow = time.time() logging.warn("LockingBlockStore: timeout locking %s", hash_key) raise LockError
def test_store(self): ss = blockstore.StreamStore(bs=self.bs, ls=self.ls, entity_set=self.cdef['Streams']) with self.cdef['Streams'].open() as streams: with self.cdef['BlockLists'].open() as blocks: stream1 = streams.new_entity() stream1['mimetype'].set_from_value("text/plain") now = TimePoint.from_now_utc() stream1['created'].set_from_value(now) stream1['modified'].set_from_value(now) streams.insert_entity(stream1) stream2 = streams.new_entity() stream2['mimetype'].set_from_value("text/plain") now = TimePoint.from_now_utc() stream2['created'].set_from_value(now) stream2['modified'].set_from_value(now) streams.insert_entity(stream2) fox = b"The quick brown fox jumped over the lazy dog" cafe = ul("Caf\xe9").encode('utf-8') ss.store_block(stream1, 0, cafe) ss.store_block(stream1, 1, fox) ss.store_block(stream2, 0, cafe) self.assertTrue(len(blocks) == 3) blocks1 = list(ss.retrieve_blocklist(stream1)) self.assertTrue(ss.retrieve_block(blocks1[0]) == cafe) self.assertTrue(ss.retrieve_block(blocks1[1]) == fox) blocks2 = list(ss.retrieve_blocklist(stream2)) self.assertTrue(ss.retrieve_block(blocks2[0]) == cafe) ss.delete_blocks(stream1, 1) # should also have deleted fox from the block store self.assertTrue(len(blocks) == 2) try: ss.retrieve_block(blocks1[1]) self.fail("Expected missing block") except blockstore.BlockMissing: pass self.assertTrue(ss.retrieve_block(blocks1[0]) == cafe) ss.delete_blocks(stream1) self.assertTrue(len(blocks) == 1) self.assertTrue(ss.retrieve_block(blocks2[0]) == cafe)
def lock(self, hash_key, timeout=60): """Acquires the lock on hash_key or raises LockError The return value is a context manager object that will automatically release the lock on hash_key when it exits. locks are not nestable, they can only be acquired once. If the lock cannot be acquired a back-off strategy is implemented using random waits up to a total maximum of *timeout* seconds. If the lock still cannot be obtained :py:class:`LockError` is raised.""" owner = "%s_%i" % (self.magic, threading.current_thread().ident) with self.entity_set.OpenCollection() as locks: tnow = time.time() tstop = tnow + timeout twait = 0 while tnow < tstop: time.sleep(twait) lock = locks.new_entity() lock['hash'].set_from_value(hash_key) lock['owner'].set_from_value(owner) lock['created'].set_from_value(TimePoint.from_now_utc()) try: locks.insert_entity(lock) return LockStoreContext(self, hash_key) except edm.ConstraintError: pass try: lock = locks[hash_key] except KeyError: # someone deleted the lock, go straight round again twait = 0 tnow = time.time() continue # has this lock expired? locktime = lock['created'].value.with_zone(zdirection=0) if locktime.get_unixtime() + self.lock_timeout < tnow: # use optimistic locking lock['owner'].set_from_value(owner) try: locks.update_entity(lock) logging.warn("LockingBlockStore removed stale lock " "on %s", hash_key) return LockStoreContext(self, hash_key) except KeyError: twait = 0 tnow = time.time() continue except edm.ConstraintError: pass twait = random.randint(0, timeout // 5) tnow = time.time() logging.warn("LockingBlockStore: timeout locking %s", hash_key) raise LockError
def flush(self): if self._bdirty: # the current block is dirty, write it out data = self._bdata[:self._btop] if data: block = self.blocks[self._bnum] if block.exists: self.ss.update_block(block, str(data)) else: self.blocks[self._bnum] = self.ss.store_block( self.stream, self._bnum, data) if self.size != self.stream['size'].value: self.stream['size'].SetFromValue(self.size) now = TimePoint.FromNowUTC() self.stream['modified'].SetFromValue(now) self.stream.Update() self._bdirty = False
def new_stream(self, mimetype=http.MediaType('application', 'octet-stream')): """Creates a new stream in the store. mimetype A :py:class:`~pyslet.rfc2616.MimeType` object Returns a stream entity which is an :py:class:`~pyslet.odata2.csdl.Entity` instance. The stream is identified by the stream entity's key which you can store elsewhere as a reference.""" with self.stream_set.OpenCollection() as streams: stream = streams.new_entity() if not isinstance(mimetype, http.MediaType): mimetype = http.MediaType.FromString(mimetype) stream['mimetype'].SetFromValue(str(mimetype)) now = TimePoint.FromNowUTC() stream['size'].SetFromValue(0) stream['created'].SetFromValue(now) stream['modified'].SetFromValue(now) streams.insert_entity(stream) return stream