def testAppend(self):
self.assertEquals([[1, 2, 3], [4, 5, 6], [7], [8]], self.run_pipeline(
common.Append([1, 2, 3], [4, 5, 6], [7], [8])).default.value)
self.assertEquals([[], [], [], []], self.run_pipeline(
common.Append([], [], [], [])).default.value)
self.assertEquals([1, 2, 3, 4, 5, 6, 7, 8], self.run_pipeline(
common.Append(1, 2, 3, 4, 5, 6, 7, 8)).default.value)
self.assertEquals([], self.run_pipeline(common.Append()).default.value)
def run(self, entity_kind, property_name, *value_list):
all_counts = []
for value in value_list:
stage = yield LongCount(entity_kind, property_name, value)
all_counts.append(stage)
yield common.Append(*all_counts)
def run(self, job_name, bucket_name, filenames):
sort_mappers = []
for i in range(len(filenames)):
filenames_only = util.strip_prefix_from_items(
"/%s/" % bucket_name, filenames[i])
sort_mapper = yield mapper_pipeline.MapperPipeline(
"%s-shuffle-sort-%s" % (job_name, str(i)),
__name__ + "._sort_records_map",
__name__ + "._BatchGCSRecordsReader",
None, {
"input_reader": {
"bucket_name": bucket_name,
"objects": filenames_only,
},
},
shards=1)
sort_mappers.append(sort_mapper)
with pipeline.After(*sort_mappers):
job_ids = yield pipeline_common.Append(
*[mapper.job_id for mapper in sort_mappers])
result = yield _CollectOutputFiles(job_ids)
with pipeline.After(result):
yield _CleanupOutputFiles(job_ids)
yield pipeline_common.Return(result)