def transform_func(iter, init_param, param):
data_tuple_list = list(iter)
data_list = [x[0] for x in data_tuple_list]
valid_list = [x[1] for x in data_tuple_list]
param["data_creator"] = make_data_creator(data_list)
param["validation_data_creator"] = make_data_creator(
valid_list)
return SparkRunner(**init_param).step(**param)
def transform_func(iter, init_param, param):
partition_data = list(iter)
# res = combine_in_partition(partition_data)
param["data_creator"] = make_data_creator(partition_data)
return SparkRunner(**init_param).predict(**param)
def transform_func(iter, init_param, param):
partition_data = list(iter)
param["data_creator"] = make_data_creator(partition_data)
return SparkRunner(**init_param).validate(**param)
def transform_func(worker, shards_ref):
params["data_creator"] = make_data_creator(shards_ref)
return worker.predict.remote(**params)
def transform_func(worker, partition_refs):
params["data_creator"] = make_data_creator(partition_refs)
return worker.validate.remote(**params)
def zip_func(worker, this_partition_refs, that_partition_refs):
params["data_creator"] = make_data_creator(
this_partition_refs)
params["validation_data_creator"] = \
make_data_creator(that_partition_refs)
return worker.step.remote(**params)