def test_progressMonitor_deprecated(mock_progressmonitor, caplog): with caplog.at_level(logging.WARNING): obj = MultiprocessingDropbox(progressMonitor=mock_progressmonitor) assert len(caplog.records) == 1 assert caplog.records[0].levelname == 'WARNING' assert 'MultiprocessingDropbox' in caplog.records[0].name assert 'deprecated.' in caplog.records[0].msg
def test_progressbar_deprecated(progressbar, caplog): with caplog.at_level(logging.WARNING): obj = MultiprocessingDropbox(progressbar=progressbar) assert len(caplog.records) == 1 assert caplog.records[0].levelname == 'WARNING' # assert 'MultiprocessingDropbox' in caplog.records[0].name # this becomes `alphatwirl.misc.deprecation` assert 'deprecated.' in caplog.records[0].msg
def obj(request, tmpdir_factory, monkeypatch): name = request.param if name == 'CommunicationChannel0': ret = CommunicationChannel0() elif name == 'CommunicationChannel-multiprocessing': dropbox = MultiprocessingDropbox() ret = CommunicationChannel(dropbox=dropbox) elif name == 'CommunicationChannel-subprocess': topdir = str(tmpdir_factory.mktemp('concurrently')) topdir = os.path.join(topdir, '_ccsp_temp') workingarea = WorkingArea(topdir=topdir, python_modules=()) dropbox = TaskPackageDropbox(workingArea=workingarea, dispatcher=SubprocessRunner()) ret = CommunicationChannel(dropbox=dropbox) ret.begin() yield ret ret.end()
def __call__(self, progressReporter = None): n = random.randint(5, 1000000) taskid = uuid.uuid4() time.sleep(random.randint(0, 3)) for i in range(n): time.sleep(0.0001) report = ProgressReport(name = self.name, done = i + 1, total = n, taskid = taskid) progressReporter.report(report) return None ##__________________________________________________________________|| progressBar = ProgressBar() if sys.stdout.isatty() else ProgressPrint() ##__________________________________________________________________|| progressMonitor = BProgressMonitor(presentation = progressBar) dropbox = MultiprocessingDropbox(nprocesses = 10, progressMonitor = progressMonitor) channel = CommunicationChannel(dropbox) progressMonitor.begin() channel.begin() channel.put(Task("loop")) channel.put(Task("another loop")) channel.put(Task("more loop")) channel.put(Task("loop loop loop")) channel.put(Task("l")) channel.put(Task("loop6")) channel.put(Task("loop7")) channel.put(Task("loop8")) channel.put(Task("loop6")) channel.receive() channel.end() progressMonitor.end()
def obj(mock_progressmonitor): ret = MultiprocessingDropbox(progressMonitor=mock_progressmonitor) ret.open() yield ret ret.terminate() ret.close()
def test_open_open_close(): obj = MultiprocessingDropbox() obj.open() obj.open() # don't create workers again obj.close()
def test_open_close(): obj = MultiprocessingDropbox() obj.open() obj.close()
def test_repr(): obj = MultiprocessingDropbox() repr(obj)
def test_init_raise(): with pytest.raises(ValueError): MultiprocessingDropbox(nprocesses=0)
def obj(): ret = MultiprocessingDropbox() ret.open() yield ret ret.terminate() ret.close()
def test_task_without_ProgressReporterno(): progressMonitor = MockProgressMonitor() obj = MultiprocessingDropbox(nprocesses=3, progressMonitor=progressMonitor) obj.open() result1 = MockResult('task1') task1 = MockTaskWithArguments(result1, 0.003) package1 = TaskPackage(task=task1, args=(111, 222, 333), kwargs={}) obj.put(package1) result2 = MockResult('task2') task2 = MockTaskWithArguments(result2, 0.001) package2 = TaskPackage(task=task2, args=(444, 555), kwargs=dict(C=666)) obj.put(package2) result3 = MockResult('task3') task3 = MockTaskWithArgumentsAndProgressReporter(result3, 0.001) package3 = TaskPackage(task=task3, args=(777, 888), kwargs=dict(C=999)) obj.put(package3) # the results in the main process don't have the attributes assert not hasattr(result1, 'A') assert not hasattr(result1, 'B') assert not hasattr(result1, 'C') assert not hasattr(result2, 'A') assert not hasattr(result2, 'B') assert not hasattr(result2, 'C') assert not hasattr(result3, 'A') assert not hasattr(result3, 'B') assert not hasattr(result3, 'C') assert not hasattr(result3, 'progressReporter') # the results returned from other processes do. returnedResults = obj.receive() assert 'task1' == returnedResults[0].data assert 111 == returnedResults[0].A assert 222 == returnedResults[0].B assert 333 == returnedResults[0].C assert 'task2' == returnedResults[1].data assert 444 == returnedResults[1].A assert 555 == returnedResults[1].B assert 666 == returnedResults[1].C assert 'task3' == returnedResults[2].data assert 777 == returnedResults[2].A assert 888 == returnedResults[2].B assert 999 == returnedResults[2].C assert isinstance(returnedResults[2].progressReporter, MockProgressReporter) obj.close()
def test_ProgressMonitor(): progressMonitor = MockProgressMonitor() obj = MultiprocessingDropbox(nprocesses=3, progressMonitor=progressMonitor) obj.open() result1 = MockResult('task1') task1 = MockTaskWithProgressReporter(result1, 0.003) package1 = TaskPackage(task=task1, args=(), kwargs={}) obj.put(package1) result2 = MockResult('task2') task2 = MockTaskWithProgressReporter(result2, 0.001) package2 = TaskPackage(task=task2, args=(), kwargs={}) obj.put(package2) # the results in the main process don't have a ProgressReporter assert not hasattr(result1, "progressReporter") assert not hasattr(result2, "progressReporter") # the results returned from other processes do. returnedResults = obj.receive() assert isinstance(returnedResults[0].progressReporter, MockProgressReporter) assert isinstance(returnedResults[1].progressReporter, MockProgressReporter) obj.close()