def with_futureproof(): executor = futureproof.ThreadPoolExecutor(max_workers=5) with futureproof.TaskManager(executor) as tm: for url in URLS: tm.submit(load_url, url, 60) for task in tm.as_completed(): print("%r page is %d bytes" % (task.args[0], len(task.result)))
def with_futureproof(): logger.info("Starting test") ex = futureproof.ThreadPoolExecutor(max_workers=2, monitor_interval=5) with futureproof.TaskManager(ex) as tm: for i in range(20): tm.submit(delayed_sum, i, 1) for task in tm.as_completed(): print(task.result)
def with_futureproof(): logger.info("Starting test") ex = futureproof.ThreadPoolExecutor(max_workers=2) with futureproof.TaskManager( ex, error_policy=futureproof.ErrorPolicyEnum. RAISE # RAISE is the default ) as tm: for i in range(50): tm.submit(flaky_sum, i, 1)
def with_futureproof(): executor = futureproof.ThreadPoolExecutor(max_workers=5) with futureproof.TaskManager( executor, error_policy=futureproof.ErrorPolicyEnum.IGNORE) as tm: for url in URLS: tm.submit(load_url, url, 60) for task in tm.completed_tasks: if isinstance(task.result, Exception): print("%r generated an exception: %s" % (task.args[0], task.result)) else: print("%r page is %d bytes" % (task.args[0], len(task.result)))
def with_futureproof(): logger.info("Starting as_completed test") ex = futureproof.ThreadPoolExecutor(max_workers=2) results = [] # the user must explictly state she wants to catch the exceptions manually with futureproof.TaskManager( ex, error_policy=futureproof.ErrorPolicyEnum.IGNORE) as tm: for i in range(10): tm.submit(flaky_sum, i, 1) for task in tm.as_completed(): if isinstance(task.result, Exception): logger.exception("Exception raised in result", exc_info=task.result) else: results.append(task.result) print(results)
def get_thread_executor(*args, request=None, **kwargs): kwargs.setdefault("max_workers", 3) if request and sys.version_info >= (3, 6): kwargs["thread_name_prefix"] = request.function.__name__ return futureproof.ThreadPoolExecutor(*args, **kwargs)