def __launch_tasks(self, ks):
# type: (List[int]) -> None
"""Execute clustering in separate threads for all given ks."""
futures = [self.__executor.submit(
self._compute_clustering,
data=self.data,
k=k,
init=['random', 'k-means++'][self.smart_init],
n_init=self.n_init,
max_iter=self.max_iterations,
silhouette=True,
) for k in ks]
watcher = FutureSetWatcher(futures)
watcher.resultReadyAt.connect(self.__clustering_complete)
watcher.progressChanged.connect(self.__progress_changed)
watcher.exceptionReadyAt.connect(self.__on_exception)
watcher.doneAll.connect(self.__commit_finished)
self.__task = Task(futures, watcher)
self.progressBarInit(processEvents=False)
self.setBlocking(True)
def __launch_tasks(self, ks):
# type: (List[int]) -> None
"""Execute clustering in separate threads for all given ks."""
futures = [self.__executor.submit(
self._compute_clustering,
data=Normalize()(self.data) if self.normalize else self.data,
k=k,
init=self.INIT_METHODS[self.smart_init][1],
n_init=self.n_init,
max_iter=self.max_iterations,
random_state=RANDOM_STATE,
) for k in ks]
watcher = FutureSetWatcher(futures)
watcher.resultReadyAt.connect(self.__clustering_complete)
watcher.progressChanged.connect(self.__progress_changed)
watcher.exceptionReadyAt.connect(self.__on_exception)
watcher.doneAll.connect(self.__commit_finished)
self.__task = Task(futures, watcher)
self.progressBarInit()
self.setInvalidated(True)
def test_watcher(self):
def spies(w):
# type: (FutureSetWatcher) -> SimpleNamespace
return SimpleNamespace(
doneAt=QSignalSpy(w.doneAt),
finishedAt=QSignalSpy(w.finishedAt),
cancelledAt=QSignalSpy(w.cancelledAt),
resultAt=QSignalSpy(w.resultReadyAt),
exceptionAt=QSignalSpy(w.exceptionReadyAt),
doneAll=QSignalSpy(w.doneAll),
)
executor = ThreadPoolExecutor(max_workers=5)
fs = [
executor.submit(lambda i: "Hello {}".format(i), i)
for i in range(10)
]
w = FutureSetWatcher(fs)
spy = spies(w)
def as_set(seq):
# type: (Iterable[list]) -> Set[tuple]
seq = list(map(tuple, seq))
set_ = set(seq)
assert len(set_) == len(seq)
return set_
self.assertTrue(spy.doneAll.wait())
expected = {(i, "Hello {}".format(i)) for i in range(10)}
self.assertSetEqual(as_set(spy.doneAt), set(enumerate(fs)))
self.assertSetEqual(as_set(spy.finishedAt), set(enumerate(fs)))
self.assertSetEqual(as_set(spy.cancelledAt), set())
self.assertSetEqual(as_set(spy.resultAt), expected)
self.assertSetEqual(as_set(spy.exceptionAt), set())
rseq = [random.randrange(0, 10) for _ in range(10)]
fs = [executor.submit(lambda i: 1 / (i % 3), i) for i in rseq]
w = FutureSetWatcher(fs)
spy = spies(w)
self.assertTrue(spy.doneAll.wait())
self.assertSetEqual(as_set(spy.doneAt), set(enumerate(fs)))
self.assertSetEqual(as_set(spy.finishedAt), set(enumerate(fs)))
self.assertSetEqual(as_set(spy.cancelledAt), set())
results = {(i, f.result())
for i, f in enumerate(fs) if not f.exception()}
exceptions = {(i, f.exception())
for i, f in enumerate(fs) if f.exception()}
assert len(results | exceptions) == len(fs)
self.assertSetEqual(as_set(spy.resultAt), results)
self.assertSetEqual(as_set(spy.exceptionAt), exceptions)
executor = ThreadPoolExecutor(max_workers=1)
ev = threading.Event()
# Block the single worker thread to ensure successful cancel for f2
f1 = executor.submit(lambda: ev.wait())
f2 = executor.submit(lambda: 42)
w = FutureSetWatcher([f1, f2])
self.assertTrue(f2.cancel())
# Unblock the worker
ev.set()
spy = spies(w)
self.assertTrue(spy.doneAll.wait())
self.assertSetEqual(as_set(spy.doneAt), {(0, f1), (1, f2)})
self.assertSetEqual(as_set(spy.finishedAt), {(0, f1)})
self.assertSetEqual(as_set(spy.cancelledAt), {(1, f2)})
self.assertSetEqual(as_set(spy.resultAt), {(0, True)})
self.assertSetEqual(as_set(spy.exceptionAt), set())
# doneAll must always be emitted after the doneAt signals.
executor = ThreadPoolExecutor(max_workers=2)
futures = [executor.submit(pow, 1000, 1000) for _ in range(100)]
watcher = FutureSetWatcher(futures)
emithistory = []
watcher.doneAt.connect(lambda i, f: emithistory.append(
("doneAt", i, f)))
watcher.doneAll.connect(lambda: emithistory.append(("doneAll", )))
spy = spies(watcher)
watcher.wait()
self.assertEqual(len(spy.doneAll), 0)
self.assertEqual(len(spy.doneAt), 0)
watcher.flush()
self.assertEqual(len(spy.doneAt), 100)
self.assertEqual(list(spy.doneAll), [[]])
self.assertSetEqual(set(emithistory[:-1]),
{("doneAt", i, f)
for i, f in enumerate(futures)})
self.assertEqual(emithistory[-1], ("doneAll", ))
# doneAll must be emitted even when on an empty futures list
watcher = FutureSetWatcher()
watcher.setFutures([])
spy = spies(watcher)
self.assertTrue(spy.doneAll.wait())
watcher = FutureSetWatcher()
watcher.setFutures([])
watcher.wait()
watcher = FutureSetWatcher()
with self.assertRaises(RuntimeError):
watcher.wait()
with unittest.mock.patch.object(
watcher, "thread",
lambda: 42), self.assertRaises(RuntimeError):
watcher.flush()
def test_watcher(self):
def spies(w):
# type: (FutureSetWatcher) -> SimpleNamespace
return SimpleNamespace(
doneAt=QSignalSpy(w.doneAt),
finishedAt=QSignalSpy(w.finishedAt),
cancelledAt=QSignalSpy(w.cancelledAt),
resultAt=QSignalSpy(w.resultReadyAt),
exceptionAt=QSignalSpy(w.exceptionReadyAt),
doneAll=QSignalSpy(w.doneAll),
)
executor = ThreadPoolExecutor(max_workers=5)
fs = [executor.submit(lambda i: "Hello {}".format(i), i)
for i in range(10)]
w = FutureSetWatcher(fs)
spy = spies(w)
def as_set(seq):
# type: (Iterable[list]) -> Set[tuple]
seq = list(map(tuple, seq))
set_ = set(seq)
assert len(set_) == len(seq)
return set_
self.assertTrue(spy.doneAll.wait())
expected = {(i, "Hello {}".format(i)) for i in range(10)}
self.assertSetEqual(as_set(spy.doneAt), set(enumerate(fs)))
self.assertSetEqual(as_set(spy.finishedAt), set(enumerate(fs)))
self.assertSetEqual(as_set(spy.cancelledAt), set())
self.assertSetEqual(as_set(spy.resultAt), expected)
self.assertSetEqual(as_set(spy.exceptionAt), set())
rseq = [random.randrange(0, 10) for _ in range(10)]
fs = [executor.submit(lambda i: 1 / (i % 3), i) for i in rseq]
w = FutureSetWatcher(fs)
spy = spies(w)
self.assertTrue(spy.doneAll.wait())
self.assertSetEqual(as_set(spy.doneAt), set(enumerate(fs)))
self.assertSetEqual(as_set(spy.finishedAt), set(enumerate(fs)))
self.assertSetEqual(as_set(spy.cancelledAt), set())
results = {(i, f.result())
for i, f in enumerate(fs) if not f.exception()}
exceptions = {(i, f.exception())
for i, f in enumerate(fs) if f.exception()}
assert len(results | exceptions) == len(fs)
self.assertSetEqual(as_set(spy.resultAt), results)
self.assertSetEqual(as_set(spy.exceptionAt), exceptions)
executor = ThreadPoolExecutor(max_workers=1)
ev = threading.Event()
# Block the single worker thread to ensure successful cancel for f2
f1 = executor.submit(lambda: ev.wait())
f2 = executor.submit(lambda: 42)
w = FutureSetWatcher([f1, f2])
self.assertTrue(f2.cancel())
# Unblock the worker
ev.set()
spy = spies(w)
self.assertTrue(spy.doneAll.wait())
self.assertSetEqual(as_set(spy.doneAt), {(0, f1), (1, f2)})
self.assertSetEqual(as_set(spy.finishedAt), {(0, f1)})
self.assertSetEqual(as_set(spy.cancelledAt), {(1, f2)})
self.assertSetEqual(as_set(spy.resultAt), {(0, True)})
self.assertSetEqual(as_set(spy.exceptionAt), set())
# doneAll must always be emitted after the doneAt signals.
executor = ThreadPoolExecutor(max_workers=2)
futures = [executor.submit(pow, 1000, 1000) for _ in range(100)]
watcher = FutureSetWatcher(futures)
emithistory = []
watcher.doneAt.connect(lambda i, f: emithistory.append(("doneAt", i, f)))
watcher.doneAll.connect(lambda: emithistory.append(("doneAll", )))
spy = spies(watcher)
watcher.wait()
self.assertEqual(len(spy.doneAll), 0)
self.assertEqual(len(spy.doneAt), 0)
watcher.flush()
self.assertEqual(len(spy.doneAt), 100)
self.assertEqual(list(spy.doneAll), [[]])
self.assertSetEqual(set(emithistory[:-1]),
{("doneAt", i, f) for i, f in enumerate(futures)})
self.assertEqual(emithistory[-1], ("doneAll",))
# doneAll must be emitted even when on an empty futures list
watcher = FutureSetWatcher()
watcher.setFutures([])
spy = spies(watcher)
self.assertTrue(spy.doneAll.wait())
watcher = FutureSetWatcher()
watcher.setFutures([])
watcher.wait()
watcher = FutureSetWatcher()
with self.assertRaises(RuntimeError):
watcher.wait()
with unittest.mock.patch.object(watcher, "thread", lambda: 42), \
self.assertRaises(RuntimeError):
watcher.flush()
def test_watcher(self):
def spies(w):
# type: (FutureSetWatcher) -> SimpleNamespace
return SimpleNamespace(
doneAt=QSignalSpy(w.doneAt),
finishedAt=QSignalSpy(w.finishedAt),
cancelledAt=QSignalSpy(w.cancelledAt),
resultAt=QSignalSpy(w.resultReadyAt),
exceptionAt=QSignalSpy(w.exceptionReadyAt),
doneAll=QSignalSpy(w.doneAll),
)
executor = ThreadPoolExecutor(max_workers=5)
fs = [
executor.submit(lambda i: "Hello {}".format(i), i)
for i in range(10)
]
w = FutureSetWatcher(fs)
spy = spies(w)
def as_set(seq):
# type: (Iterable[list]) -> Set[tuple]
seq = list(map(tuple, seq))
set_ = set(seq)
assert len(set_) == len(seq)
return set_
self.assertTrue(spy.doneAll.wait())
expected = {(i, "Hello {}".format(i)) for i in range(10)}
self.assertSetEqual(as_set(spy.doneAt), set(enumerate(fs)))
self.assertSetEqual(as_set(spy.finishedAt), set(enumerate(fs)))
self.assertSetEqual(as_set(spy.cancelledAt), set())
self.assertSetEqual(as_set(spy.resultAt), expected)
self.assertSetEqual(as_set(spy.exceptionAt), set())
rseq = [random.randrange(0, 10) for _ in range(10)]
fs = [executor.submit(lambda i: 1 / (i % 3), i) for i in rseq]
w = FutureSetWatcher(fs)
spy = spies(w)
self.assertTrue(spy.doneAll.wait())
self.assertSetEqual(as_set(spy.doneAt), set(enumerate(fs)))
self.assertSetEqual(as_set(spy.finishedAt), set(enumerate(fs)))
self.assertSetEqual(as_set(spy.cancelledAt), set())
results = {(i, f.result())
for i, f in enumerate(fs) if not f.exception()}
exceptions = {(i, f.exception())
for i, f in enumerate(fs) if f.exception()}
assert len(results | exceptions) == len(fs)
self.assertSetEqual(as_set(spy.resultAt), results)
self.assertSetEqual(as_set(spy.exceptionAt), exceptions)
executor = ThreadPoolExecutor(max_workers=1)
ev = threading.Event()
# Block the single worker thread to ensure successful cancel for f2
f1 = executor.submit(lambda: ev.wait())
f2 = executor.submit(lambda: 42)
w = FutureSetWatcher([f1, f2])
self.assertTrue(f2.cancel())
# Unblock the worker
ev.set()
spy = spies(w)
self.assertTrue(spy.doneAll.wait())
self.assertSetEqual(as_set(spy.doneAt), {(0, f1), (1, f2)})
self.assertSetEqual(as_set(spy.finishedAt), {(0, f1)})
self.assertSetEqual(as_set(spy.cancelledAt), {(1, f2)})
self.assertSetEqual(as_set(spy.resultAt), {(0, True)})
self.assertSetEqual(as_set(spy.exceptionAt), set())