def __create_binary_process(self, left: _DTable, right: _DTable, func):
left_op = self.__create_storage_locator_from_dtable(left)
right_op = self.__create_storage_locator_from_dtable(right)
task_info = self.__create_task_info(func=func, is_in_place_computing=left.get_in_place_computing())
return processor_pb2.BinaryProcess(info=task_info,
left=left_op,
right=right_op,
conf=processor_pb2.ProcessConf(namingPolicy=self.eggroll_context.get_naming_policy().name))
def __create_binary_process(self, left: _DTable, right: _DTable, func,
session):
left_op = self.__create_storage_locator_from_dtable(left)
right_op = self.__create_storage_locator_from_dtable(right)
task_info = self.__create_task_info(
func=func, is_in_place_computing=left.get_in_place_computing())
return processor_pb2.BinaryProcess(
info=task_info,
left=left_op,
right=right_op,
session=self.eggroll_session.to_protobuf())
def process_wrapper(req_type, func, result, req):
try:
req_pb = processor_pb2.UnaryProcess(
) if req_type == "UnaryProcess" else processor_pb2.BinaryProcess(
)
req_pb.ParseFromString(req)
#TODO context serialize?
func(req_pb, None)
result.put("ok")
except:
err_str = traceback.format_exc()
LOGGER.error(err_str)
result.put("error:" + err_str)
def join(self, left, right, func):
func_id, func_bytes = self.serialize_and_hash_func(func)
results = []
res = None
for partition in range(left.partition):
l_op = EggRoll.__get_storage_locator(left, partition)
r_op = EggRoll.__get_storage_locator(right, partition)
binary_p = processor_pb2.BinaryProcess(
left=l_op,
right=r_op,
info=processor_pb2.TaskInfo(task_id=self.job_id,
function_id=func_id,
function_bytes=func_bytes))
proc_id = partition % len(self.proc_list)
channel, stub = self.proc_list[proc_id]
results.append(stub.join.future(binary_p))
for r in results:
res = r.result()
return _DTable(self, res.type, res.namespace, res.name, left.partition)