def test_ProgressMonitor():
progressMonitor = MockProgressMonitor()
obj = MultiprocessingDropbox(nprocesses=3, progressMonitor=progressMonitor)
obj.open()
result1 = MockResult('task1')
task1 = MockTaskWithProgressReporter(result1, 0.003)
package1 = TaskPackage(task=task1, args=(), kwargs={})
obj.put(package1)
result2 = MockResult('task2')
task2 = MockTaskWithProgressReporter(result2, 0.001)
package2 = TaskPackage(task=task2, args=(), kwargs={})
obj.put(package2)
# the results in the main process don't have a ProgressReporter
assert not hasattr(result1, "progressReporter")
assert not hasattr(result2, "progressReporter")
# the results returned from other processes do.
returnedResults = obj.receive()
assert isinstance(returnedResults[0].progressReporter,
MockProgressReporter)
assert isinstance(returnedResults[1].progressReporter,
MockProgressReporter)
obj.close()
def test_put_multiple(obj, dropbox):
obj.begin()
task1 = mock.Mock(name='task1')
task2 = mock.Mock(name='task2')
task3 = mock.Mock(name='task3')
task4 = mock.Mock(name='task4')
obj.put_multiple([
task1,
dict(task=task2, args=(123, 'ABC'), kwargs={'A': 34}),
dict(task=task3, kwargs={'B': 123}),
dict(task=task4, args=(222, 'def')),
])
expected = [
mock.call([
TaskPackage(task=task1, args=(), kwargs={}),
TaskPackage(task=task2, args=(123, 'ABC'), kwargs={'A': 34}),
TaskPackage(task=task3, args=(), kwargs={'B': 123}),
TaskPackage(task=task4, args=(222, 'def'), kwargs={}),
])
]
assert expected == dropbox.put_multiple.call_args_list
obj.end()
def test_task_without_ProgressReporterno():
progressMonitor = MockProgressMonitor()
obj = MultiprocessingDropbox(nprocesses=3, progressMonitor=progressMonitor)
obj.open()
result1 = MockResult('task1')
task1 = MockTaskWithArguments(result1, 0.003)
package1 = TaskPackage(task=task1, args=(111, 222, 333), kwargs={})
obj.put(package1)
result2 = MockResult('task2')
task2 = MockTaskWithArguments(result2, 0.001)
package2 = TaskPackage(task=task2, args=(444, 555), kwargs=dict(C=666))
obj.put(package2)
result3 = MockResult('task3')
task3 = MockTaskWithArgumentsAndProgressReporter(result3, 0.001)
package3 = TaskPackage(task=task3, args=(777, 888), kwargs=dict(C=999))
obj.put(package3)
# the results in the main process don't have the attributes
assert not hasattr(result1, 'A')
assert not hasattr(result1, 'B')
assert not hasattr(result1, 'C')
assert not hasattr(result2, 'A')
assert not hasattr(result2, 'B')
assert not hasattr(result2, 'C')
assert not hasattr(result3, 'A')
assert not hasattr(result3, 'B')
assert not hasattr(result3, 'C')
assert not hasattr(result3, 'progressReporter')
# the results returned from other processes do.
returnedResults = obj.receive()
assert 'task1' == returnedResults[0].data
assert 111 == returnedResults[0].A
assert 222 == returnedResults[0].B
assert 333 == returnedResults[0].C
assert 'task2' == returnedResults[1].data
assert 444 == returnedResults[1].A
assert 555 == returnedResults[1].B
assert 666 == returnedResults[1].C
assert 'task3' == returnedResults[2].data
assert 777 == returnedResults[2].A
assert 888 == returnedResults[2].B
assert 999 == returnedResults[2].C
assert isinstance(returnedResults[2].progressReporter,
MockProgressReporter)
obj.close()
def test_put(obj, dropbox):
obj.begin()
task1 = mock.MagicMock(name='task1')
obj.put(task1)
task2 = mock.MagicMock(name='task2')
obj.put(task2, 123, 'ABC', A=34)
expected = [
mock.call(TaskPackage(task=task1, args=(), kwargs={})),
mock.call(TaskPackage(task=task2, args=(123, 'ABC'), kwargs={'A': 34})),
]
assert expected == dropbox.put.call_args_list
obj.end()
def package_rel_path_task_raise(workingarea_path):
ret = 'task_00009.p.gz'
package = TaskPackage(task=task_raise, args=[], kwargs={})
path = os.path.join(workingarea_path, ret)
with gzip.open(path, 'wb') as f:
pickle.dump(package, f, protocol=pickle.HIGHEST_PROTOCOL)
return ret
def test_run_task_null(self):
package = TaskPackage(task=task_null, args=[], kwargs={})
package_path = os.path.join(self.tempdir_path, 'task_00009.p.gz')
with gzip.open(package_path, 'wb') as f:
pickle.dump(package, f, protocol=pickle.HIGHEST_PROTOCOL)
f.close()
proc = subprocess.Popen(
[os.path.join('.', 'run.py'), 'task_00009.p.gz'],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=self.tempdir_path,
env=self.env)
stdout, stderr = proc.communicate()
def package4():
task = MockTask(name='task4', time=0.002)
args = ()
kwargs = dict(ABC='abc', DEF='def')
return TaskPackage(task=task, args=args, kwargs=kwargs)
def package3():
task = MockTask(name='task3', time=0.005)
args = (33, 44)
kwargs = {}
return TaskPackage(task=task, args=args, kwargs=kwargs)
def package2():
task = MockTask(name='task2', time=0.001)
args = ()
kwargs = {}
return TaskPackage(task=task, args=args, kwargs=kwargs)
def package1():
task = MockTask(name='task1', time=0.010)
args = (111, 222)
kwargs = dict(A='abc', B='def')
return TaskPackage(task=task, args=args, kwargs=kwargs)
def package1():
args = (111, 222)
kwargs = dict(A='abc', B='def')
return TaskPackage(task=mocktask, args=args, kwargs=kwargs)
def package4():
result4 = MockResult('task4')
task4 = MockTask(result4, 0.002)
return TaskPackage(task=task4, args=(), kwargs={})
def package3():
result3 = MockResult('task3')
task3 = MockTask(result3, 0.005)
return TaskPackage(task=task3, args=(), kwargs={})
def package2():
result2 = MockResult('task2')
task2 = MockTask(result2, 0.001)
return TaskPackage(task=task2, args=(), kwargs={})
def package1():
result1 = MockResult('task1')
task1 = MockTask(result1, 0.010)
return TaskPackage(task=task1, args=(), kwargs={})