def __deliver_responses(): import time registered_deliveries = r.scard('deliveries') deliveries_ready = r.scard('deliveries:ready') log.info("""Delivery daemon started: - Deliveries: {} - Ready: {}""".format(registered_deliveries, deliveries_ready)) log.info('Delivery daemon started') futures = {} while True: ready = r.smembers('deliveries:ready') for rid in ready: if rid not in futures: log.info('Response delivery of request {} is ready. Preparing...'.format(rid)) futures[rid] = thp.submit(__deliver_response, rid) for obsolete_rid in set.difference(set(futures.keys()), ready): if obsolete_rid in futures and futures[obsolete_rid].done(): del futures[obsolete_rid] sent = r.smembers('deliveries:sent') for rid in sent: r.srem('deliveries:ready', rid) r.srem('deliveries', rid) try: response = build_response(rid) response.sink.remove() log.info('Request {} was sent and cleared'.format(rid)) except AttributeError: log.warning('Request number {} was deleted by other means'.format(rid)) pass r.delete('deliveries:sent') time.sleep(1)
log.info( """Fragment daemon setup: - On-demand threshold: {} - Minimum sync time: {} - Maximum concurrent collectors: {} - Maximum concurrent fragments: {}""".format( ON_DEMAND_TH, MIN_SYNC, N_COLLECTORS, MAX_CONCURRENT_FRAGMENTS ) ) thp = ThreadPoolExecutor(max_workers=min(8, MAX_CONCURRENT_FRAGMENTS)) log.info("Cleaning fragment locks...") fragment_locks = r.keys("*lock*") for flk in fragment_locks: r.delete(flk) log.info("Cleaning fragment pulling flags...") fragment_pullings = r.keys("fragments:*:pulling") for fpk in fragment_pullings: r.delete(fpk) log.info("Releasing registered fragments...") fragment_consumers = r.keys("fragments:*:consumers") for fck in fragment_consumers: r.delete(fck) class FragmentPlugin(object): __plugins = []