def run_distribute(in_path): """ Run the composition of csv_file_consumer and information tap with the csv files in the input directory, and collect the results from each file and merge them together, printing both kinds of results. """ results = distribute_run_to_runners( certain_kind_tap, in_path, reader=i_get_csv_data, batch_size=3) fruit_results = [] metal_results = [] for fruits, metals in results: for fruit in fruits: fruit_results.append(fruit) for metal in metals: metal_results.append(metal) print("=== fruits ===") for fruit in fruit_results: print(fruit) print("=== metals ===") for metal in metal_results: print(metal)
def test_default_reader(self): """ Ensure results have the items_func applied to them and were processed in batches of batch_size. """ from karld.run_together import distribute_run_to_runners input_path = os.path.join(os.path.dirname(__file__), "test_data", "things_kinds", "people.json") test_results = distribute_run_to_runners( json_values, input_path, batch_size=1) self.assertEqual( test_results, [ [u'John'], [u'Sally'] ])