def run_distribute_multi(in_dir): """ Run the composition of csv_file_consumer and information tap with the csv files in the input directory, and collect the results from each file and merge them together, printing both kinds of results. """ results = distribute_multi_run_to_runners( certain_kind_tap, in_dir, reader=i_get_csv_data, batch_size=3, filter_func=is_file_csv) fruit_results = [] metal_results = [] for fruits, metals in results: for fruit in fruits: fruit_results.append(fruit) for metal in metals: metal_results.append(metal) print("=== fruits ===") for fruit in fruit_results: print(fruit) print("=== metals 䤗 ===") for metal in metal_results: print(metal)
def test_default_reader(self): """ Ensure results have given items_func, collect_first, applied to them. """ import karld from karld.run_together import distribute_multi_run_to_runners input_path = os.path.join(os.path.dirname(__file__), "test_data", "things_kinds") test_results = distribute_multi_run_to_runners( collect_first, in_dir=input_path, batch_size=10, filter_func=karld.io.is_file_csv) list_results = list(test_results) self.assertEqual(len(list_results[0]), 10) self.assertEqual(len(list_results[1]), 5) if is_py3(): self.assertEqual( sorted(chain.from_iterable(list_results)), [87, 97, 99, 99, 99, 100, 105, 109, 111, 112, 112, 114, 116, 116, 116] ) else: self.assertEqual( sorted(chain.from_iterable(list_results)), ['W', 'a', 'c', 'c', 'c', 'd', 'i', 'm', 'o', 'p', 'p', 'r', 't', 't', 't'], )
def test_csv_reader(self): """ Ensure results have the function applied to them. Ensure only csv files are read. Ensure the batches of results match batch_size. Ensure the given function is applied to the items of a batch. """ import karld from karld.run_together import distribute_multi_run_to_runners input_path = os.path.join(os.path.dirname(__file__), "test_data", "things_kinds") test_results = distribute_multi_run_to_runners( collect_first, in_dir=input_path, reader=karld.io.i_get_csv_data, batch_size=3, filter_func=karld.io.is_file_csv) list_results = list(test_results) self.assertEqual(len(list_results), 5) self.assertEqual(len(list_results[0]), 3) self.assertEqual( sorted(chain.from_iterable(list_results)), [u'W\u0104\u017b', u'apple', u'cat', u'celery', u'cheese', u'dr\xf3\u017ck\u0105', u'iron', u'mushroom', u'orange', u'peach', u'pear', u'ruby', u'titanium', u'tomato', u'topaz'] )