def run(self, task: Task, hosts: List[Host]) -> AggregatedResult:
"""
This is where the magic happens
"""
# we instantiate the aggregated result
result = AggregatedResult(task.name)
with tqdm(
total=len(hosts),
desc="progress",
) as progress:
with ThreadPoolExecutor(max_workers=self.num_workers) as pool:
futures = {
pool.submit(task.copy().start, host): host
for host in hosts
}
for future in as_completed(futures):
worker_result = future.result()
result[worker_result.host.name] = worker_result
progress.update()
if worker_result.failed:
tqdm.write(f"{worker_result.host.name}: failure")
else:
tqdm.write(f"{worker_result.host.name}: success")
return result
def run(self, task: Task, hosts: List[Host]) -> AggregatedResult:
result = AggregatedResult(task.name)
futures = []
with ThreadPoolExecutor(self.num_workers) as pool:
for host in hosts:
future = pool.submit(task.copy().start, host)
futures.append(future)
for future in futures:
worker_result = future.result()
result[worker_result.host.name] = worker_result
return result
def run(self, task: Task, hosts: List[Host]) -> AggregatedResult:
connectors_q = queue.Queue()
work_q = queue.Queue()
result = AggregatedResult(task.name)
# enqueue hosts in connectors queue
for host in hosts:
connectors_q.put(
(task.copy(), host, {"connection_retry": 0, "task_retry": 0}, result)
)
# start connectors threads
connector_threads = []
for i in range(self.num_connectors):
t = threading.Thread(target=self.connector, args=(connectors_q, work_q))
t.start()
connector_threads.append(t)
# start worker threads
worker_threads = []
for i in range(self.num_workers):
t = threading.Thread(target=self.worker, args=(connectors_q, work_q))
t.start()
worker_threads.append(t)
# wait until all hosts completed task or timeout reached
start_time = time.time()
while True:
with LOCK:
hosts_no_result = [h.name for h in hosts if h.name not in result]
if hosts_no_result == []:
break
if time.time() - start_time > self.task_timeout:
log.error("RetryRunner task '{}', '{}' seconds wait timeout reached, hosts that did not return results '{}'".format(
task.name, self.task_timeout, hosts_no_result
)
)
break
time.sleep(0.1)
# block until all queues empty
connectors_q.join()
work_q.join()
# stop connector threads
for i in range(self.num_connectors):
connectors_q.put(None)
for t in connector_threads:
t.join()
# stop worker threads
for i in range(self.num_workers):
work_q.put(None)
for t in worker_threads:
t.join()
# delete queues and threads
del(connectors_q, work_q, connector_threads, worker_threads)
return result
def _run_parallel(self, task: Task, hosts, num_workers, **kwargs):
result = AggregatedResult(kwargs.get("name") or task.name)
pool = Pool(processes=num_workers)
result_pool = [
pool.apply_async(task.copy().start, args=(h, self)) for h in hosts
]
pool.close()
pool.join()
for rp in result_pool:
r = rp.get()
result[r.host.name] = r
return result
def run(self, task: Task, hosts: List[Host]) -> AggregatedResult:
result = AggregatedResult(task.name)
greenlets = []
pool = Pool(self.num_workers)
for host in hosts:
greenlet = pool.spawn(task.copy().start, host)
greenlets.append(greenlet)
pool.join()
for greenlet in greenlets:
worker_result = greenlet.get()
result[worker_result.host.name] = worker_result
return result
def _run_parallel(
self,
task: Task,
hosts: List["Host"],
num_workers: int,
**kwargs: Dict[str, Any],
) -> AggregatedResult:
agg_result = AggregatedResult(kwargs.get("name") or task.name)
futures = []
with ThreadPoolExecutor(num_workers) as pool:
for host in hosts:
future = pool.submit(task.copy().start, host, self)
futures.append(future)
for future in futures:
worker_result = future.result()
agg_result[worker_result.host.name] = worker_result
return agg_result
def run(self, task: Task, hosts: List[Host]) -> AggregatedResult:
"""
This is where the magic happens
"""
# we instantiate the aggregated result
result = AggregatedResult(task.name)
with ThreadPoolExecutor(max_workers=self.num_workers) as pool:
futures = {
pool.submit(task.copy().start, host): host
for host in hosts
}
for future in as_completed(futures):
worker_result = future.result()
result[worker_result.host.name] = worker_result
if worker_result.failed:
print(f'{worker_result.host.name} - fail')
else:
print(f'{worker_result.host.name} - success')
return result
def run(self, task: Task, hosts: List[Host]) -> AggregatedResult:
"""
This is where the magic happens
"""
# we instantiate the aggregated result
result = AggregatedResult(task.name)
with Progress(
"[progress.description]{task.description}",
BarColumn(),
"[progress.percentage]{task.completed:>3.0f}/{task.total}",
) as progress:
num_hosts = len(hosts)
total = progress.add_task("[cyan]Completed...", total=num_hosts)
successful = progress.add_task("[green]Successful...",
total=num_hosts)
changed = progress.add_task("[orange3]Changed...", total=num_hosts)
error = progress.add_task("[red]Failed...", total=num_hosts)
with ThreadPoolExecutor(max_workers=self.num_workers) as pool:
futures = {
pool.submit(task.copy().start, host): host
for host in hosts
}
for future in as_completed(futures):
worker_result = future.result()
result[worker_result.host.name] = worker_result
progress.update(total, advance=1)
if worker_result.failed:
progress.update(error, advance=1)
progress.print(
f"[red]{worker_result.host.name}: failure")
else:
progress.update(successful, advance=1)
progress.print(
f"[green]{worker_result.host.name}: success")
if worker_result.changed:
progress.update(changed, advance=1)
return result
def run(self, task: Task, hosts: List[Host]) -> AggregatedResult:
"""
This is where the magic happens
"""
# first we create the root object with all the device groups in it
self.root = sort_hosts(hosts)
# we instantiate the aggregated result
result = AggregatedResult(task.name)
# when sending the tasks to the pool we will store the futures here
futures = []
with ThreadPoolExecutor(self.num_workers) as pool:
while self.root.pending():
# for as long as we have pending objects
# we execute the task over a batch of devices and store
# the futures
for host in self.root.batch():
future = pool.submit(task.copy().start, host)
futures.append(future)
# we process the futures
while futures:
future = futures.pop(0)
worker_result = future.result()
result[worker_result.host.name] = worker_result
if worker_result.failed:
self.root.fail(worker_result.host,
worker_result[-1].exception)
else:
self.root.complete(worker_result.host)
time.sleep(1)
return result
def run(self, task: Task, hosts: List[Host]) -> AggregatedResult:
result = AggregatedResult(task.name)
for host in hosts:
result[host.name] = task.copy().start(host)
return result
def _run_serial(self, task: Task, hosts, **kwargs):
result = AggregatedResult(kwargs.get("name") or task.name)
for host in hosts:
result[host.name] = task.copy().start(host, self)
return result
def run(self, task: Task, hosts: List[Host]) -> AggregatedResult:
connectors_q = queue.Queue()
work_q = queue.Queue()
stop_event = threading.Event()
task_timeout_event = threading.Event()
task_timeout_timer = threading.Timer(self.task_timeout,
lambda: task_timeout_event.set())
result = AggregatedResult(task.name)
connector_threads = []
worker_threads = []
# enqueue hosts in connectors queue
for host in hosts:
connectors_q.put((task.copy(), host, {
"connection_retry": 0,
"task_retry": 0
}, result))
# start connectors threads
for i in range(self.num_connectors):
t = threading.Thread(
target=connector,
args=(
stop_event,
connectors_q,
work_q,
self.connect_backoff,
self.connect_splay,
self.connect_retry,
self.jumphosts_connections,
),
)
t.start()
connector_threads.append(t)
# start worker threads
for i in range(self.num_workers):
t = threading.Thread(
target=worker,
args=(
stop_event,
connectors_q,
work_q,
self.task_backoff,
self.task_splay,
self.task_retry,
self.connect_retry,
self.reconnect_on_fail,
),
)
t.start()
worker_threads.append(t)
# wait until all hosts completed task or timeout reached
task_timeout_timer.start()
while not task_timeout_event.is_set():
with LOCK:
hosts_no_result = [
h.name for h in hosts if h.name not in result
]
if hosts_no_result == []:
task_timeout_timer.cancel()
break
time.sleep(0.1)
else:
log.warning(
"RetryRunner task '{}', '{}'s task_timeout reached, hosts no results '{}'"
.format(task.name, self.task_timeout, hosts_no_result))
# block until all queues empty
connectors_q.join()
work_q.join()
# stop and delete connectors and workers threads
stop_event.set()
while connector_threads:
_ = connector_threads.pop().join()
while worker_threads:
_ = worker_threads.pop().join()
return result
