def delete_blocks(self, stream, from_num=0):
blocks = list(self.retrieve_blocklist(stream))
filter = core.BinaryExpression(core.Operator.eq)
filter.AddOperand(core.PropertyExpression('hash'))
hash_value = edm.EDMValue.NewSimpleValue(edm.SimpleType.String)
filter.AddOperand(core.LiteralExpression(hash_value))
# filter is: hash eq <hash_value>
with self.block_set.OpenCollection() as base_coll:
for block in blocks:
if from_num and block['num'].value < from_num:
continue
hash_key = block['hash'].value
with self.ls.lock(hash_key):
del base_coll[block.key()]
# is this hash key used anywhere?
hash_value.set_from_value(hash_key)
base_coll.set_filter(filter)
if len(base_coll) == 0:
# remove orphan block from block store
self.bs.delete(hash_key)
def CleanupForever(memCache):
"""Runs a loop continuously cleaning up expired items"""
now=edm.DateTimeValue()
expires=core.PropertyExpression(u"Expires")
t=core.LiteralExpression(now)
filter=core.BinaryExpression(core.Operator.lt)
filter.operands.append(expires)
filter.operands.append(t)
while True:
now.SetFromValue(iso.TimePoint.FromNowUTC())
logging.info("Cleanup thread running at %s",str(now.value))
with memCache.OpenCollection() as cacheEntries:
cacheEntries.Filter(filter)
expiredList=list(cacheEntries)
if expiredList:
logging.info("Cleaning %i cache entries",len(expiredList))
for expired in expiredList:
del cacheEntries[expired]
cacheEntries.Filter(None)
logging.info("Cleanup complete, %i cache entries remain",len(cacheEntries))
time.sleep(CLEANUP_SLEEP)
def update_block(self, block, data):
hash_key = block['hash'].value
new_hash = self.bs.key(data)
if new_hash == hash_key:
return
filter = core.BinaryExpression(core.Operator.eq)
filter.AddOperand(core.PropertyExpression('hash'))
hash_value = edm.EDMValue.NewSimpleValue(edm.SimpleType.String)
filter.AddOperand(core.LiteralExpression(hash_value))
# filter is: hash eq <hash_value>
with self.block_set.OpenCollection() as base_coll:
with self.ls.lock(hash_key), self.ls.lock(new_hash):
self.bs.store(data)
block['hash'].SetFromValue(new_hash)
base_coll.update_entity(block)
# is the old hash key used anywhere?
hash_value.SetFromValue(hash_key)
base_coll.set_filter(filter)
if len(base_coll) == 0:
# remove orphan block from block store
self.bs.delete(hash_key)