def test_progressMonitor_deprecated(mock_progressmonitor, caplog):
with caplog.at_level(logging.WARNING):
obj = MultiprocessingDropbox(progressMonitor=mock_progressmonitor)
assert len(caplog.records) == 1
assert caplog.records[0].levelname == 'WARNING'
assert 'MultiprocessingDropbox' in caplog.records[0].name
assert 'deprecated.' in caplog.records[0].msg
def test_progressbar_deprecated(progressbar, caplog):
with caplog.at_level(logging.WARNING):
obj = MultiprocessingDropbox(progressbar=progressbar)
assert len(caplog.records) == 1
assert caplog.records[0].levelname == 'WARNING'
# assert 'MultiprocessingDropbox' in caplog.records[0].name # this becomes `alphatwirl.misc.deprecation`
assert 'deprecated.' in caplog.records[0].msg
def obj(request, tmpdir_factory, monkeypatch):
name = request.param
if name == 'CommunicationChannel0':
ret = CommunicationChannel0()
elif name == 'CommunicationChannel-multiprocessing':
dropbox = MultiprocessingDropbox()
ret = CommunicationChannel(dropbox=dropbox)
elif name == 'CommunicationChannel-subprocess':
topdir = str(tmpdir_factory.mktemp('concurrently'))
topdir = os.path.join(topdir, '_ccsp_temp')
workingarea = WorkingArea(topdir=topdir, python_modules=())
dropbox = TaskPackageDropbox(workingArea=workingarea,
dispatcher=SubprocessRunner())
ret = CommunicationChannel(dropbox=dropbox)
ret.begin()
yield ret
ret.end()
def __call__(self, progressReporter = None):
n = random.randint(5, 1000000)
taskid = uuid.uuid4()
time.sleep(random.randint(0, 3))
for i in range(n):
time.sleep(0.0001)
report = ProgressReport(name = self.name, done = i + 1, total = n, taskid = taskid)
progressReporter.report(report)
return None
##__________________________________________________________________||
progressBar = ProgressBar() if sys.stdout.isatty() else ProgressPrint()
##__________________________________________________________________||
progressMonitor = BProgressMonitor(presentation = progressBar)
dropbox = MultiprocessingDropbox(nprocesses = 10, progressMonitor = progressMonitor)
channel = CommunicationChannel(dropbox)
progressMonitor.begin()
channel.begin()
channel.put(Task("loop"))
channel.put(Task("another loop"))
channel.put(Task("more loop"))
channel.put(Task("loop loop loop"))
channel.put(Task("l"))
channel.put(Task("loop6"))
channel.put(Task("loop7"))
channel.put(Task("loop8"))
channel.put(Task("loop6"))
channel.receive()
channel.end()
progressMonitor.end()
def obj(mock_progressmonitor):
ret = MultiprocessingDropbox(progressMonitor=mock_progressmonitor)
ret.open()
yield ret
ret.terminate()
ret.close()
def test_open_open_close():
obj = MultiprocessingDropbox()
obj.open()
obj.open() # don't create workers again
obj.close()
def test_open_close():
obj = MultiprocessingDropbox()
obj.open()
obj.close()
def test_repr():
obj = MultiprocessingDropbox()
repr(obj)
def test_open_open_close():
obj = MultiprocessingDropbox()
obj.open()
obj.open() # don't create workers again
obj.close()
def test_open_close():
obj = MultiprocessingDropbox()
obj.open()
obj.close()
def test_init_raise():
with pytest.raises(ValueError):
MultiprocessingDropbox(nprocesses=0)
def obj(mock_progressmonitor):
ret = MultiprocessingDropbox(progressMonitor=mock_progressmonitor)
ret.open()
yield ret
ret.terminate()
ret.close()
def obj():
ret = MultiprocessingDropbox()
ret.open()
yield ret
ret.terminate()
ret.close()
def test_task_without_ProgressReporterno():
progressMonitor = MockProgressMonitor()
obj = MultiprocessingDropbox(nprocesses=3, progressMonitor=progressMonitor)
obj.open()
result1 = MockResult('task1')
task1 = MockTaskWithArguments(result1, 0.003)
package1 = TaskPackage(task=task1, args=(111, 222, 333), kwargs={})
obj.put(package1)
result2 = MockResult('task2')
task2 = MockTaskWithArguments(result2, 0.001)
package2 = TaskPackage(task=task2, args=(444, 555), kwargs=dict(C=666))
obj.put(package2)
result3 = MockResult('task3')
task3 = MockTaskWithArgumentsAndProgressReporter(result3, 0.001)
package3 = TaskPackage(task=task3, args=(777, 888), kwargs=dict(C=999))
obj.put(package3)
# the results in the main process don't have the attributes
assert not hasattr(result1, 'A')
assert not hasattr(result1, 'B')
assert not hasattr(result1, 'C')
assert not hasattr(result2, 'A')
assert not hasattr(result2, 'B')
assert not hasattr(result2, 'C')
assert not hasattr(result3, 'A')
assert not hasattr(result3, 'B')
assert not hasattr(result3, 'C')
assert not hasattr(result3, 'progressReporter')
# the results returned from other processes do.
returnedResults = obj.receive()
assert 'task1' == returnedResults[0].data
assert 111 == returnedResults[0].A
assert 222 == returnedResults[0].B
assert 333 == returnedResults[0].C
assert 'task2' == returnedResults[1].data
assert 444 == returnedResults[1].A
assert 555 == returnedResults[1].B
assert 666 == returnedResults[1].C
assert 'task3' == returnedResults[2].data
assert 777 == returnedResults[2].A
assert 888 == returnedResults[2].B
assert 999 == returnedResults[2].C
assert isinstance(returnedResults[2].progressReporter,
MockProgressReporter)
obj.close()
def test_ProgressMonitor():
progressMonitor = MockProgressMonitor()
obj = MultiprocessingDropbox(nprocesses=3, progressMonitor=progressMonitor)
obj.open()
result1 = MockResult('task1')
task1 = MockTaskWithProgressReporter(result1, 0.003)
package1 = TaskPackage(task=task1, args=(), kwargs={})
obj.put(package1)
result2 = MockResult('task2')
task2 = MockTaskWithProgressReporter(result2, 0.001)
package2 = TaskPackage(task=task2, args=(), kwargs={})
obj.put(package2)
# the results in the main process don't have a ProgressReporter
assert not hasattr(result1, "progressReporter")
assert not hasattr(result2, "progressReporter")
# the results returned from other processes do.
returnedResults = obj.receive()
assert isinstance(returnedResults[0].progressReporter,
MockProgressReporter)
assert isinstance(returnedResults[1].progressReporter,
MockProgressReporter)
obj.close()