def post_processing(self, release_index):
processors = [
self._update_records_with_user_info,
self._update_commits_with_merge_date,
functools.partial(self._update_records_with_releases, release_index),
self._update_blueprints_with_mention_info,
self._determine_core_contributors,
self._update_members_company_name,
self._update_marks_with_disagreement,
]
pipeline_processor = utils.make_pipeline_processor(processors)
self.runtime_storage_inst.set_records(pipeline_processor(self.runtime_storage_inst.get_all_records))
def post_processing(self, release_index):
processors = [
self._update_records_with_user_info,
self._update_commits_with_merge_date,
functools.partial(self._update_records_with_releases,
release_index),
self._update_commits_with_module_alias,
self._determine_core_contributors,
self._update_marks_with_disagreement,
]
pipeline_processor = utils.make_pipeline_processor(processors)
self.runtime_storage_inst.set_records(
pipeline_processor(self.runtime_storage_inst.get_all_records))
def test_pipeline_processor(self):
counter = dict(n=0)
consumed = []
log = mock.Mock()
def get_all_items():
for i in range(5):
counter['n'] += 1
yield i
def single_pass_uno():
log('single_pass_uno:begin')
def pass_1(s):
yield s
yield pass_1
log('single_pass_uno:end')
def single_pass_duo():
log('single_pass_duo:begin')
def pass_1(s):
yield s + 10
yield pass_1
log('single_pass_duo:end')
def double_pass():
log('double_pass:begin')
r = set()
def pass_1(s):
if s % 2:
r.add(s)
yield pass_1
log('double_pass:middle')
def pass_2(s):
if s in r:
yield s * 100
yield pass_2
log('double_pass:end')
def consume(r):
for x in r:
consumed.append(x)
processors = [single_pass_uno, double_pass, single_pass_duo]
pipeline_processor = utils.make_pipeline_processor(processors)
consume(pipeline_processor(get_all_items))
self.assertEqual(10, counter['n']) # twice by 5 elements
expected = [0, 10, 1, 11, 2, 12, 3, 13, 4, 14, 100, 300]
self.assertEqual(expected, consumed)
log.assert_has_calls([
mock.call('single_pass_uno:begin'),
mock.call('double_pass:begin'),
mock.call('single_pass_duo:begin'),
mock.call('single_pass_uno:end'),
mock.call('double_pass:middle'),
mock.call('single_pass_duo:end'),
mock.call('double_pass:end'),
])
def test_pipeline_processor(self):
counter = dict(n=0)
consumed = []
log = mock.Mock()
def get_all_items():
for i in range(5):
counter['n'] += 1
yield i
def single_pass_uno():
log('single_pass_uno:begin')
def pass_1(s):
yield s
yield pass_1
log('single_pass_uno:end')
def single_pass_duo():
log('single_pass_duo:begin')
def pass_1(s):
yield s + 10
yield pass_1
log('single_pass_duo:end')
def double_pass():
log('double_pass:begin')
r = set()
def pass_1(s):
if s % 2:
r.add(s)
yield pass_1
log('double_pass:middle')
def pass_2(s):
if s in r:
yield s * 100
yield pass_2
log('double_pass:end')
def consume(r):
for x in r:
consumed.append(x)
processors = [single_pass_uno, double_pass, single_pass_duo]
pipeline_processor = utils.make_pipeline_processor(processors)
consume(pipeline_processor(get_all_items))
self.assertEqual(10, counter['n']) # twice by 5 elements
expected = [0, 10, 1, 11, 2, 12, 3, 13, 4, 14, 100, 300]
self.assertEqual(expected, consumed)
log.assert_has_calls([
mock.call('single_pass_uno:begin'),
mock.call('double_pass:begin'),
mock.call('single_pass_duo:begin'),
mock.call('single_pass_uno:end'),
mock.call('double_pass:middle'),
mock.call('single_pass_duo:end'),
mock.call('double_pass:end'),
])
