def convert_to_protobuf(sender, workflow, **kwargs):
"""Converts an execution element and its data to a protobuf message.
Args:
sender (execution element): The execution element object that is sending the data.
workflow (Workflow): The workflow which is sending the event
kwargs (dict, optional): A dict of extra fields, such as data, callback_name, etc.
Returns:
(str): The newly formed protobuf object, serialized as a string to send over the ZMQ socket.
"""
event = kwargs['event']
data = kwargs['data'] if 'data' in kwargs else None
packet = Message()
packet.event_name = event.name
if event.event_type == EventType.workflow:
convert_workflow_to_proto(packet, workflow, data)
elif event.event_type == EventType.action:
if event == WalkoffEvent.ConsoleLog:
convert_log_message_to_protobuf(packet, sender, workflow, **kwargs)
elif event == WalkoffEvent.SendMessage:
convert_send_message_to_protobuf(packet, sender, workflow,
**kwargs)
else:
convert_action_to_proto(packet, sender, workflow, data)
elif event.event_type in (EventType.branch, EventType.condition,
EventType.transform,
EventType.conditonalexpression):
convert_branch_transform_condition_to_proto(packet, sender, workflow)
packet_bytes = packet.SerializeToString()
return packet_bytes
def convert_to_protobuf(sender, workflow_execution_uid='', **kwargs):
"""Converts an execution element and its data to a protobuf message.
Args:
sender (execution element): The execution element object that is sending the data.
workflow_execution_uid (str, optional): The execution UID of the Workflow under which this execution
element falls. It is not required and defaults to an empty string, but it is highly recommended
so that the LoadBalancer can keep track of the Workflow's execution.
kwargs (dict, optional): A dict of extra fields, such as data, callback_name, etc.
Returns:
The newly formed protobuf object, serialized as a string to send over the ZMQ socket.
"""
event = kwargs['event']
data = kwargs['data'] if 'data' in kwargs else None
packet = Message()
packet.event_name = event.name
if event.event_type == EventType.workflow:
convert_workflow_to_proto(packet, sender, workflow_execution_uid, data)
elif event.event_type == EventType.action:
if event == WalkoffEvent.SendMessage:
convert_send_message_to_protobuf(packet, sender, workflow_execution_uid, **kwargs)
else:
convert_action_to_proto(packet, sender, workflow_execution_uid, data)
elif event.event_type in (EventType.branch, EventType.condition, EventType.transform):
convert_branch_transform_condition_to_proto(packet, sender, workflow_execution_uid)
packet_bytes = packet.SerializeToString()
return packet_bytes
def to_event_callback(message_bytes):
"""Converts a message to an event callback message
"""
message_outer = Message()
message_outer.ParseFromString(message_bytes)
callback_name = message_outer.event_name
if message_outer.type == Message.WORKFLOWPACKET:
message = message_outer.workflow_packet
elif message_outer.type == Message.ACTIONPACKET:
message = message_outer.action_packet
elif message_outer.type == Message.USERMESSAGE:
message = message_outer.message_packet
elif message_outer.type == Message.LOGMESSAGE:
message = message_outer.logging_packet
else:
message = message_outer.general_packet
sender = None
if hasattr(message, "sender"):
sender = MessageToDict(message.sender,
preserving_proto_field_name=True)
elif hasattr(message, "workflow"):
sender = MessageToDict(message.workflow,
preserving_proto_field_name=True)
event = WalkoffEvent.get_event_from_name(callback_name)
if event is not None:
data = ProtobufWorkflowResultsConverter._format_callback_data(
event, message, sender)
return event, sender, data
else:
logger.error('Unknown callback {} sent'.format(callback_name))
return None, None, None
def _send_callback(self, message_bytes):
message_outer = Message()
message_outer.ParseFromString(message_bytes)
callback_name = message_outer.event_name
if message_outer.type == Message.WORKFLOWPACKET:
message = message_outer.workflow_packet
elif message_outer.type == Message.ACTIONPACKET:
message = message_outer.action_packet
elif message_outer.type == Message.USERMESSAGE:
message = message_outer.message_packet
elif message_outer.type == Message.LOGMESSAGE:
message = message_outer.logging_packet
else:
message = message_outer.general_packet
if hasattr(message, "sender"):
sender = MessageToDict(message.sender, preserving_proto_field_name=True)
elif hasattr(message, "workflow"):
sender = MessageToDict(message.workflow, preserving_proto_field_name=True)
event = WalkoffEvent.get_event_from_name(callback_name)
if event is not None:
data = self._format_data(event, message)
with self.current_app.app_context():
event.send(sender, data=data)
if event in [WalkoffEvent.WorkflowShutdown, WalkoffEvent.WorkflowAborted]:
self._increment_execution_count()
else:
logger.error('Unknown callback {} sent'.format(callback_name))
def receive_results(self):
"""Keep receiving results from execution elements over a ZMQ socket, and trigger the callbacks.
"""
while True:
if self.thread_exit:
break
try:
message_bytes = self.results_sock.recv(zmq.NOBLOCK)
except zmq.ZMQError:
gevent.sleep(0.1)
continue
message_outer = Message()
message_outer.ParseFromString(message_bytes)
callback_name = message_outer.event_name
if message_outer.type == Message.WORKFLOWPACKET:
message = message_outer.workflow_packet
elif message_outer.type == Message.ACTIONPACKET:
message = message_outer.action_packet
elif message_outer.type == Message.USERMESSAGE:
message = message_outer.message_packet
else:
message = message_outer.general_packet
sender = MessageToDict(message.sender,
preserving_proto_field_name=True)
event = WalkoffEvent.get_event_from_name(callback_name)
if event is not None:
if event.requires_data():
if event != WalkoffEvent.SendMessage:
data = json.loads(message.additional_data)
else:
data = format_message_event_data(message)
event.send(sender, data=data)
else:
event.send(sender)
if event == WalkoffEvent.WorkflowShutdown:
self.workflows_executed += 1
else:
logger.error('Unknown callback {} sent'.format(callback_name))
self.results_sock.close()
return
def send_callback(self, message_bytes):
message_outer = Message()
message_outer.ParseFromString(message_bytes)
callback_name = message_outer.event_name
if message_outer.type == Message.WORKFLOWPACKET:
message = message_outer.workflow_packet
elif message_outer.type == Message.ACTIONPACKET:
message = message_outer.action_packet
elif message_outer.type == Message.USERMESSAGE:
message = message_outer.message_packet
else:
message = message_outer.general_packet
sender = MessageToDict(message.sender,
preserving_proto_field_name=True)
event = WalkoffEvent.get_event_from_name(callback_name)
if event is not None:
if event.event_type != EventType.workflow:
data = {
'workflow':
MessageToDict(message.workflow,
preserving_proto_field_name=True)
}
else:
data = {}
if event.requires_data():
if event != WalkoffEvent.SendMessage:
data['data'] = json.loads(message.additional_data)
else:
data['message'] = format_message_event_data(message)
event.send(sender, data=data)
if event in [
WalkoffEvent.WorkflowShutdown, WalkoffEvent.WorkflowAborted
]:
self._increment_execution_count()
else:
logger.error('Unknown callback {} sent'.format(callback_name))
def event_to_protobuf(sender, workflow_ctx, **kwargs):
"""Converts an execution element and its data to a protobuf message.
Args:
sender (execution element): The execution element object that is sending the data.
workflow_ctx (WorkflowExecutionContext): The workflow which is sending the event
kwargs (dict, optional): A dict of extra fields, such as data, callback_name, etc.
Returns:
(str): The newly formed protobuf object, serialized as a string to send over the ZMQ socket.
"""
event = kwargs['event']
data = kwargs['data'] if 'data' in kwargs else None
packet = Message()
if 'user' in kwargs:
packet.user = kwargs['user']
packet.event_name = event.name
if event.event_type == EventType.workflow:
ProtobufWorkflowResultsConverter._convert_workflow_to_proto(packet, workflow_ctx, data)
elif event.event_type == EventType.action:
if event == WalkoffEvent.ConsoleLog:
ProtobufWorkflowResultsConverter._convert_log_message_to_protobuf(packet, sender, workflow_ctx,
**kwargs)
elif event == WalkoffEvent.SendMessage:
ProtobufWorkflowResultsConverter._convert_send_message_to_protobuf(packet, sender, workflow_ctx,
**kwargs)
else:
ProtobufWorkflowResultsConverter._convert_action_to_proto(packet, sender, workflow_ctx, data)
elif event.event_type in (
EventType.branch, EventType.condition, EventType.transform, EventType.conditonalexpression):
ProtobufWorkflowResultsConverter._convert_branch_transform_condition_to_proto(packet, sender, workflow_ctx)
elif event == WalkoffEvent.WorkerReady:
packet.type = Message.WORKERPACKET
packet.worker_packet.id = sender['id']
packet_bytes = packet.SerializeToString()
return packet_bytes
def event_to_protobuf(sender, workflow_ctx, **kwargs):
"""Converts an execution element and its data to a protobuf message.
Args:
sender (execution element): The execution element object that is sending the data.
workflow_ctx (WorkflowExecutionContext): The workflow which is sending the event
kwargs (dict, optional): A dict of extra fields, such as data, callback_name, etc.
Returns:
(str): The newly formed protobuf object, serialized as a string to send over the ZMQ socket.
"""
event = kwargs['event']
data = kwargs['data'] if 'data' in kwargs else None
packet = Message()
if 'user' in kwargs:
packet.user = kwargs['user']
packet.event_name = event.name
if event.event_type == EventType.workflow:
ProtobufWorkflowResultsConverter._convert_workflow_to_proto(
packet, workflow_ctx, data)
elif event.event_type == EventType.action:
if event == WalkoffEvent.ConsoleLog:
ProtobufWorkflowResultsConverter._convert_log_message_to_protobuf(
packet, sender, workflow_ctx, **kwargs)
elif event == WalkoffEvent.SendMessage:
ProtobufWorkflowResultsConverter._convert_send_message_to_protobuf(
packet, sender, workflow_ctx, **kwargs)
else:
ProtobufWorkflowResultsConverter._convert_action_to_proto(
packet, sender, workflow_ctx, data)
elif event.event_type in (EventType.branch, EventType.condition,
EventType.transform,
EventType.conditonalexpression):
ProtobufWorkflowResultsConverter._convert_branch_transform_condition_to_proto(
packet, sender, workflow_ctx)
elif event == WalkoffEvent.WorkerReady:
packet.type = Message.WORKERPACKET
packet.worker_packet.id = sender['id']
packet_bytes = packet.SerializeToString()
return packet_bytes
def test_send_message(self, mock_send):
self.controller._send_message(Message().SerializeToString())
self.assert_message_sent(mock_send, Message().SerializeToString())