def post_processing(self, release_index): processors = [ self._update_records_with_user_info, self._update_commits_with_merge_date, functools.partial(self._update_records_with_releases, release_index), self._update_blueprints_with_mention_info, self._determine_core_contributors, self._update_members_company_name, self._update_marks_with_disagreement, ] pipeline_processor = utils.make_pipeline_processor(processors) self.runtime_storage_inst.set_records(pipeline_processor(self.runtime_storage_inst.get_all_records))
def post_processing(self, release_index): processors = [ self._update_records_with_user_info, self._update_commits_with_merge_date, functools.partial(self._update_records_with_releases, release_index), self._update_commits_with_module_alias, self._determine_core_contributors, self._update_marks_with_disagreement, ] pipeline_processor = utils.make_pipeline_processor(processors) self.runtime_storage_inst.set_records( pipeline_processor(self.runtime_storage_inst.get_all_records))
def test_pipeline_processor(self): counter = dict(n=0) consumed = [] log = mock.Mock() def get_all_items(): for i in range(5): counter['n'] += 1 yield i def single_pass_uno(): log('single_pass_uno:begin') def pass_1(s): yield s yield pass_1 log('single_pass_uno:end') def single_pass_duo(): log('single_pass_duo:begin') def pass_1(s): yield s + 10 yield pass_1 log('single_pass_duo:end') def double_pass(): log('double_pass:begin') r = set() def pass_1(s): if s % 2: r.add(s) yield pass_1 log('double_pass:middle') def pass_2(s): if s in r: yield s * 100 yield pass_2 log('double_pass:end') def consume(r): for x in r: consumed.append(x) processors = [single_pass_uno, double_pass, single_pass_duo] pipeline_processor = utils.make_pipeline_processor(processors) consume(pipeline_processor(get_all_items)) self.assertEqual(10, counter['n']) # twice by 5 elements expected = [0, 10, 1, 11, 2, 12, 3, 13, 4, 14, 100, 300] self.assertEqual(expected, consumed) log.assert_has_calls([ mock.call('single_pass_uno:begin'), mock.call('double_pass:begin'), mock.call('single_pass_duo:begin'), mock.call('single_pass_uno:end'), mock.call('double_pass:middle'), mock.call('single_pass_duo:end'), mock.call('double_pass:end'), ])
def test_pipeline_processor(self): counter = dict(n=0) consumed = [] log = mock.Mock() def get_all_items(): for i in range(5): counter['n'] += 1 yield i def single_pass_uno(): log('single_pass_uno:begin') def pass_1(s): yield s yield pass_1 log('single_pass_uno:end') def single_pass_duo(): log('single_pass_duo:begin') def pass_1(s): yield s + 10 yield pass_1 log('single_pass_duo:end') def double_pass(): log('double_pass:begin') r = set() def pass_1(s): if s % 2: r.add(s) yield pass_1 log('double_pass:middle') def pass_2(s): if s in r: yield s * 100 yield pass_2 log('double_pass:end') def consume(r): for x in r: consumed.append(x) processors = [single_pass_uno, double_pass, single_pass_duo] pipeline_processor = utils.make_pipeline_processor(processors) consume(pipeline_processor(get_all_items)) self.assertEqual(10, counter['n']) # twice by 5 elements expected = [0, 10, 1, 11, 2, 12, 3, 13, 4, 14, 100, 300] self.assertEqual(expected, consumed) log.assert_has_calls([ mock.call('single_pass_uno:begin'), mock.call('double_pass:begin'), mock.call('single_pass_duo:begin'), mock.call('single_pass_uno:end'), mock.call('double_pass:middle'), mock.call('single_pass_duo:end'), mock.call('double_pass:end'), ])