def send_message_to_wayang(self):
connections = {}
sources = []
sinks = []
operators = []
for operator in self.originals:
if not operator.is_unary():
raise PywyException(
"the not unary operator are not supported".format(
type(operator), operator))
if operator.is_operator():
connections[operator] = self.add_proto_unary_operator(operator)
operators.append(connections[operator])
elif operator.is_source():
connections[operator] = self.add_proto_source_operator(
operator)
sources.append(connections[operator])
elif operator.is_sink():
connections[operator] = self.add_proto_sink_operator(operator)
sinks.append(connections[operator])
else:
raise PywyException(
"the type {} for the operator {} is not supported {}".
format(type(operator), operator,
WayangJVMTextFileSink.mro()))
for operator in self.originals:
current = connections[operator]
for ele in operator.previous:
current.predecessors.append(connections.get(ele).id)
for ele in operator.nexts:
current.successors.append(connections.get(ele).id)
plan_configuration = pwb.WayangPlanProto()
plan = pwb.PlanProto()
plan.sources.extend(sources)
plan.operators.extend(operators)
plan.sinks.extend(sinks)
plan.input = pwb.PlanProto.string
plan.output = pwb.PlanProto.string
ctx = pwb.ContextProto()
ctx.platforms.extend([pwb.ContextProto.PlatformProto.java])
plan_configuration.plan.CopyFrom(plan)
plan_configuration.context.CopyFrom(ctx)
print("plan!")
print(plan_configuration)
msg_bytes = plan_configuration.SerializeToString()
msg_64 = base64.b64encode(msg_bytes)
logging.debug(msg_bytes)
# response = requests.get("http://localhost:8080/plan/create/fromfile")
data = {'message': msg_64}
response = requests.post("http://localhost:8080/plan/create", data)
logging.debug(response)
def get_type_flatmap_function(call: FlatmapFunction) -> (type, type):
sig = signature(call)
if len(sig.parameters) != 1:
raise PywyException(
"the parameters for the FlatmapFunction are distinct than one, {}".
format(str(sig.parameters)))
if type(sig.return_annotation) != type(Iterable):
raise PywyException(
"the return for the FlatmapFunction is not Iterable, {}".format(
str(sig.return_annotation)))
keys = list(sig.parameters.keys())
return sig.parameters[
keys[0]].annotation, sig.return_annotation.__args__[0]
def provide_dispatchable(self,
do_wrapper: bool = False) -> WayangJVMOperator:
if self.operator is None:
raise PywyException("The operator was not define")
if do_wrapper:
self.operator.udf = self.udf
return self.operator
def get_type_function(call: Function) -> (type, type):
sig = signature(call)
if len(sig.parameters) != 1:
raise PywyException(
"the parameters for the Function are distinct than one, {}".format(
str(sig.parameters)))
keys = list(sig.parameters.keys())
return sig.parameters[keys[0]].annotation, sig.return_annotation
def get_type_predicate(call: Predicate) -> type:
sig = signature(call)
if len(sig.parameters) != 1:
raise PywyException(
"the parameters for the Predicate are distinct than one, {}".
format(str(sig.parameters)))
keys = list(sig.parameters.keys())
return sig.parameters[keys[0]].annotation
def concatenate(function_a: Callable, function_b: Callable):
if function_a is None:
raise PywyException("the function_a can't be None")
if function_b is None:
return function_a
def executable(iterable):
return function_a(function_b(iterable))
return executable
def validate_vector(self,
vect: List['WayangJVMOperator'],
index: int,
op: 'WayangJVMOperator' = None):
if op is None:
op = self
if vect is None or len(vect) == 0:
vect = [None] * (index + 1)
if len(vect) < index:
vect.extend([None for i in range(index + 1 - len(vect))])
if vect[index] is not None:
raise PywyException(
'the position in the index "{}" is already in use for "{}" in the operator "{}"'
.format(index, vect[index], op))
return vect