def __deliver_responses():
import time
registered_deliveries = r.scard('deliveries')
deliveries_ready = r.scard('deliveries:ready')
log.info("""Delivery daemon started:
- Deliveries: {}
- Ready: {}""".format(registered_deliveries, deliveries_ready))
log.info('Delivery daemon started')
futures = {}
while True:
ready = r.smembers('deliveries:ready')
for rid in ready:
if rid not in futures:
log.info('Response delivery of request {} is ready. Preparing...'.format(rid))
futures[rid] = thp.submit(__deliver_response, rid)
for obsolete_rid in set.difference(set(futures.keys()), ready):
if obsolete_rid in futures and futures[obsolete_rid].done():
del futures[obsolete_rid]
sent = r.smembers('deliveries:sent')
for rid in sent:
r.srem('deliveries:ready', rid)
r.srem('deliveries', rid)
try:
response = build_response(rid)
response.sink.remove()
log.info('Request {} was sent and cleared'.format(rid))
except AttributeError:
log.warning('Request number {} was deleted by other means'.format(rid))
pass
r.delete('deliveries:sent')
time.sleep(1)
log.info(
"""Fragment daemon setup:
- On-demand threshold: {}
- Minimum sync time: {}
- Maximum concurrent collectors: {}
- Maximum concurrent fragments: {}""".format(
ON_DEMAND_TH, MIN_SYNC, N_COLLECTORS, MAX_CONCURRENT_FRAGMENTS
)
)
thp = ThreadPoolExecutor(max_workers=min(8, MAX_CONCURRENT_FRAGMENTS))
log.info("Cleaning fragment locks...")
fragment_locks = r.keys("*lock*")
for flk in fragment_locks:
r.delete(flk)
log.info("Cleaning fragment pulling flags...")
fragment_pullings = r.keys("fragments:*:pulling")
for fpk in fragment_pullings:
r.delete(fpk)
log.info("Releasing registered fragments...")
fragment_consumers = r.keys("fragments:*:consumers")
for fck in fragment_consumers:
r.delete(fck)
class FragmentPlugin(object):
__plugins = []
