def run_distribute(in_path):
"""
Run the composition of csv_file_consumer and information tap
with the csv files in the input directory, and collect
the results from each file and merge them together,
printing both kinds of results.
"""
results = distribute_run_to_runners(
certain_kind_tap,
in_path, reader=i_get_csv_data, batch_size=3)
fruit_results = []
metal_results = []
for fruits, metals in results:
for fruit in fruits:
fruit_results.append(fruit)
for metal in metals:
metal_results.append(metal)
print("=== fruits ===")
for fruit in fruit_results:
print(fruit)
print("=== metals ===")
for metal in metal_results:
print(metal)
def test_default_reader(self):
"""
Ensure results have the items_func applied to them
and were processed in batches of batch_size.
"""
from karld.run_together import distribute_run_to_runners
input_path = os.path.join(os.path.dirname(__file__),
"test_data",
"things_kinds",
"people.json")
test_results = distribute_run_to_runners(
json_values,
input_path,
batch_size=1)
self.assertEqual(
test_results,
[
[u'John'],
[u'Sally']
])
