def receive(self):
debug_msg("Python - Wait to receive message in loop.")
message = self._receive_queue.get()
debug_msg("Python - Received message in loop: " + str(message))
if message == self._monitor.poison_pill:
raise RuntimeError("Message receive loop terminated.")
return message
def deserialize(bytes):
durationstr = bytes.decode("utf-8")
hidx = durationstr.find('H');
hours = 0
if hidx >= 0:
hours = int(durationstr[:hidx])
durationstr = durationstr[hidx+1:]
midx = durationstr.find('M')
minutes = 0
if midx >= 0:
minutes = int(durationstr[:midx])
durationstr = durationstr[midx+1:]
sidx = durationstr.find('S')
seconds = 0
millis = 0
if sidx >= 0:
pidx = durationstr.find('.')
if pidx > 0:
seconds = int(durationstr[:pidx])
millistr = durationstr[pidx+1:sidx]
while len(millistr) < 3:
millistr += '0'
millis = int(millistr)
if seconds < 0:
millis *= -1
else:
seconds = int(durationstr[:sidx])
debug_util.debug_msg('Decoded: ' + bytes.decode("utf-8") + ' to ' + str(datetime.timedelta(hours=hours, minutes=minutes, seconds=seconds, milliseconds=millis)))
return datetime.timedelta(hours=hours, minutes=minutes, seconds=seconds, milliseconds=millis)
def execute(self, source_code, initiating_message_id=None):
"""
Execute the given source code
"""
output = StringIO()
error = StringIO()
# log to stdout and output variable simultaneously
backup_std_out = sys.stdout
sys.stdout = PythonKernelBase._Logger(sys.stdout, output)
# FIXME: This is dangerous!
self._exec_env[
'python_messaging_initiating_message_id'] = initiating_message_id
# run execute with the provided source code
try:
exec(source_code, self._exec_env, self._exec_env)
except Exception:
# Print failing source code to simplify debugging.
debug_msg("Source code that caused failure:\n" + source_code)
backup_std_error = sys.stderr
sys.stderr = error
traceback.print_exc()
sys.stderr.flush()
sys.stderr = backup_std_error
sys.stdout = backup_std_out
return [output.getvalue(), error.getvalue()]
def set(self, result):
with self._condition:
import debug_util
debug_util.debug_msg("Python - Set result: " + str(result))
self._result = result
self._is_done = True
self._condition.notify()
def _handle_custom_message(self, message, response_message_id_supplier, response_consumer, result_consumer,
workspace):
if message.category == "my-nested-request-from-java":
request_message_type = "first-request"
response_payload = None
else:
message_type = message.get_header_field(AbstractTaskHandler.FIELD_KEY_MESSAGE_TYPE)
if message_type == "first":
self._response_string_payload += PayloadDecoder(message.payload).get_next_string()
request_message_type = "second-request"
response_payload = None
elif message_type == "second":
workspace.unregister_task_handler("my-nested-request-from-java")
result_consumer(None)
self._response_string_payload += PayloadDecoder(message.payload).get_next_string()
request_message_type = AbstractTaskHandler.MESSAGE_TYPE_SUCCESS
response_payload = PayloadEncoder().put_string(self._response_string_payload).payload
else:
error_message = "Message type '" + message_type + "' is not recognized. Illegal state."
debug_util.debug_msg(error_message)
raise RuntimeError(error_message)
response_consumer[0] = Message(workspace._commands._messaging.create_next_message_id(), str(message.id),
payload=response_payload,
additional_options={AbstractTaskHandler.FIELD_KEY_MESSAGE_TYPE:
request_message_type})
return True
def set_exception(self, exception):
with self._condition:
if self._exception is None and exception is not None:
debug_msg("Python - Set exception: " + str(exception))
self._exception = exception
self._is_done = True
self._condition.notify()
def deserialize_from_bytes(self, data_frame, column_serializers):
"""
Deserialize all cells in the provided data frame from a bytes representation (inplace).
@param data_frame a pandas.DataFrame containing columns to deserialize
@param column_serializers dict containing column names present in data_frame as keys and deserializer_ids as
values. A deserializer_id should be the id of the java extension point on which the
deserializer is registered. Each column identified by the dict keys is deserialized
using the deserializer provided by the TypeExtensionManager for the given
deserializer_id.
"""
# print('Data frame: ' + str(data_frame) + '\nserializers: ' + str(column_serializers) + '\n')
for column in column_serializers:
deserializer = self._type_extension_manager.get_deserializer_by_id(
column_serializers[column])
for i in range(len(data_frame)):
if debug_util.is_debug_enabled():
lastp = -1
if (i * 100 / len(data_frame)) % 5 == 0 and int(
i * 100 / len(data_frame)) != lastp:
debug_util.debug_msg(
str(i * 100 / len(data_frame)) +
' percent done (deserialize)')
# lastp = int(i * 100/len(data_frame))
col_idx = data_frame.columns.get_loc(column)
# Using bracket accessor is necessary here for ensuring that there are no unwanted type conversions.
value = data_frame[column][data_frame.index[i]]
if isinstance(value, numpy.float64) and numpy.isnan(value):
value = None
if value:
if isinstance(value, list):
new_list = []
for inner_value in value:
if isinstance(inner_value, numpy.float64
) and numpy.isnan(inner_value):
inner_value = None
if inner_value:
new_list.append(
deserializer.deserialize(inner_value))
else:
new_list.append(None)
data_frame.iat[i, col_idx] = new_list
elif isinstance(value, set):
new_set = set()
for inner_value in value:
if isinstance(inner_value, numpy.float64
) and numpy.isnan(inner_value):
inner_value = None
if inner_value:
new_set.add(
deserializer.deserialize(inner_value))
else:
new_set.add(None)
data_frame.iat[i, col_idx] = new_set
else:
data_frame.iat[i, col_idx] = deserializer.deserialize(
value)
else:
data_frame.iat[i, col_idx] = None
def _handle_custom_message(self, message, response_message_id_supplier, response_consumer, result_consumer,
workspace):
debug_msg("Python - Respond to message: " + str(message))
response = self._respond(message, response_message_id_supplier(), workspace)
debug_msg("Python - Responded to message: " + str(message) + ", response: " + str(response))
response_consumer[0] = response
result_consumer(None) # We are done after the response is sent.
return True
def run(self):
try:
self._runnable()
except BaseException as ex:
import debug_util
# debug_util.breakpoint()
debug_util.debug_msg("An exception occurred while running a task. Cause: " + str(ex), exc_info=True)
raise
def set_exc_info(self, exc_info):
with self._condition:
if self._exc_info is None and exc_info is not None and exc_info[
0] is not None:
debug_msg("Python - Set exception: " + str(exc_info[1]))
self._exc_info = exc_info
self._is_done = True
self._condition.notify()
def handle(self, message):
debug_msg("Python - Enqueue message for task, message: " +
str(message) + ", initiating message: " +
str(self._initiating_message))
self._received_messages.put(message)
debug_msg("Python - Now calling 'run'.")
self.run() # Start task if not already running.
return True
def handle(self, message):
message_handler = self._message_handlers.get(message.category)
if message_handler is not None:
debug_msg("Python - Distribute message: " + str(message))
message_handler.handle(message)
else:
raise RuntimeError(
"Message '" + str(message) +
"' cannot be distributed. No matching handler available for category '"
+ message.category + "'.")
def run(self):
try:
self._runnable()
except BaseException as ex:
self.set_exc_info(sys.exc_info())
debug_msg(
"An exception occurred while running a task. Cause: " +
str(ex),
exc_info=True)
raise
def get(self):
with self._condition:
__counter = 0
while not self.is_done:
import debug_util
__counter += 1
debug_util.debug_msg("Python - Wait for result (" + str(__counter) + ").")
self._condition.wait()
result = self._result
return result
def _loop(self):
while self.is_running:
message = None
try:
message = self._receiver.receive()
self._distributor.handle(message)
except Exception as ex:
debug_msg("Failed to distribute message " + (
"'" + str(message) + "' " if message is not None else "") + "from Java. Cause: " + str(ex))
raise
def receive(self):
header_size = self._read_size()
payload_size = self._read_size()
header = self._read_data(header_size).decode('utf-8')
if payload_size > 0:
payload = self._read_data(payload_size)
else:
payload = None
message = Message.create(header, payload)
debug_msg("Python - Received message: " + str(message))
return message
def test_request_from_java_to_python(workspace):
import debug_util
debug_util.debug_msg("Entering test_request_from_java_to_python.")
class _RequestHandler(AbstractRequestHandler):
def _respond(self, request, response_message_id, workspace):
workspace.unregister_task_handler("my-request-from-java")
payload = PayloadEncoder().put_string("my-response-from-python").payload
return AbstractRequestHandler._create_response(request, response_message_id, response_payload=payload)
workspace.register_task_handler("my-request-from-java", _RequestHandler())
def _respond(self, request, response_message_id, workspace):
source_code = PayloadDecoder(request.payload).get_next_string()
debug_msg('Executing:\n' + source_code)
output, error = workspace.execute(source_code, request.id)
debug_msg('Execution done.')
response_payload = PayloadEncoder().put_string(output).put_string(
error).payload
return AbstractRequestHandler._create_response(
request, response_message_id, response_payload=response_payload)
def test_request_from_java_that_causes_request_from_python(workspace):
import debug_util
debug_util.debug_msg("Entering test_request_from_java_that_causes_request_from_python.")
class _RequestHandler(AbstractRequestHandler):
def _respond(self, request, response_message_id, workspace):
workspace.unregister_task_handler("my-request-from-java-that-causes-a-request-from-python")
result_from_request = trigger_request_from_python(workspace)
payload = PayloadEncoder().put_string(result_from_request + "-made-the-task-succeed").payload
return AbstractRequestHandler._create_response(request, response_message_id, True, payload)
workspace.register_task_handler("my-request-from-java-that-causes-a-request-from-python", _RequestHandler())
def send(self, message):
debug_msg("Python - Send message: " + str(message))
header = message.header.encode('utf-8')
self._write_size(len(header))
payload = message.payload
if payload:
self._write_size(len(payload))
else:
self._write_size(0)
self._write_data(header)
if payload:
self._write_data(payload)
def serialize(timedelta_obj):
microstr = str(timedelta_obj.microseconds // 1000)
while len(microstr) < 3:
microstr = '0' + microstr
durationstr = str(timedelta_obj.days * 24 +
timedelta_obj.seconds // 3600) + 'H ' + str(
(timedelta_obj.seconds % 3600) // 60) + 'm ' + str(
timedelta_obj.seconds %
60) + '.' + microstr + 's'
debug_util.debug_msg('Serializing timedelta: ' + str(timedelta_obj) +
' as ' + durationstr)
return durationstr.encode("utf-8")
def report_exception(self, exception, message=None):
if self._exception is None and exception is not None:
self._exception = exception
try:
if message is not None:
error_message = message + " Cause: " + str(exception)
else:
error_message = "An exception occurred: " + str(exception)
debug_msg(error_message, exc_info=True)
except BaseException:
pass
self.report_close()
def get(self):
with self._condition:
__counter = 0
while not self.is_done:
__counter += 1
debug_msg("Python - Wait for result (" + str(__counter) +
").")
self._condition.wait()
exc_info = self._exc_info
result = self._result
if exc_info is not None:
Task.FutureTask._reraise_exception(exc_info)
else:
return result
def get(self):
with self._condition:
__counter = 0
while not self.is_done:
__counter += 1
debug_msg("Python - Wait for result (" + str(__counter) +
").")
self._condition.wait()
exception = self._exception
result = self._result
if exception is not None:
raise exception
else:
return result
def serialize_objects_to_bytes(self, data_frame, column_serializers):
"""
Serialize all cells in the provided data frame to a bytes representation (inplace).
@param data_frame a pandas.DataFrame containing columns to serialize
@param column_serializers dict containing column names present in data_frame as keys and serializer_ids as
values.
A serializer_id should be the id of the java extension point on which the serializer
is registered. Each column identified by the dict keys is serialized using the
serializer provided by the TypeExtensionManager for the given serializer_id.
"""
for column in column_serializers:
serializer = self._type_extension_manager.get_serializer_by_id(
column_serializers[column])
col_idx = data_frame.columns.get_loc(column)
if data_frame[column].dtype != 'object':
data_frame[column] = data_frame[column].astype('object')
for i in range(len(data_frame)):
if debug_util.is_debug_enabled():
lastp = -1
if (i * 100 / len(data_frame)) % 5 == 0 and int(
i * 100 / len(data_frame)) != lastp:
debug_util.debug_msg(
str(i * 100 / len(data_frame)) +
' percent done (serialize)')
# lastp = int(i * 100/len(data_frame))
# Using bracket acessor is necessary here for ensuring that there are
# no unwanted type conversions
value = data_frame[column][data_frame.index[i]]
if value is not None:
if isinstance(value, list):
new_list = []
for inner_value in value:
if inner_value is None:
new_list.append(None)
else:
new_list.append(
serializer.serialize(inner_value))
data_frame.iat[i, col_idx] = new_list
elif isinstance(value, set):
new_set = set()
for inner_value in value:
if inner_value is None:
new_set.add(None)
else:
new_set.add(serializer.serialize(inner_value))
data_frame.iat[i, col_idx] = new_set
else:
data_frame.iat[i,
col_idx] = serializer.serialize(value)
def test_request_from_python_to_java(workspace):
import debug_util
debug_util.debug_msg("Entering test_request_from_python_to_java.")
class _TaskHandler(AbstractTaskHandler):
def _handle_success_message(self, message):
import debug_util
debug_util.debug_msg("Inside handler of Java response.")
return PayloadDecoder(message.payload).get_next_string()
my_task = workspace._commands.create_task(_TaskHandler(), Message(
workspace._commands._messaging.create_next_message_id(), "my-request-from-python"))
print(my_task.get()) # 'flush' keyword argument is not supported by Python 2.
sys.stdout.flush()
def start(self):
if not self._is_running:
if self._is_closed:
raise RuntimeError('Python kernel is closed and cannot be restarted.')
self._is_running = True
debug_msg("Connect.")
self._connection = self._connect(('localhost', int(sys.argv[1])))
debug_msg("Create executors.")
self._execute_thread_executor = self._create_execute_thread_executor()
self._executor = self._create_executor()
debug_msg("Create Python commands.")
self._commands = PythonCommands(self._create_messaging(self._connection), self)
self._setup_builtin_request_handlers()
debug_msg("Create type extension manager.")
self._type_extension_manager = TypeExtensionManager(self._commands)
# Start commands/messaging system once everything is set up.
debug_msg("Start Python commands.")
self._commands.start()
def _handle_custom_message(self, message, response_message_id_supplier,
response_consumer, result_consumer, workspace):
response_message_id = response_message_id_supplier()
try:
debug_msg("Python - Respond to message: " + str(message))
response = self._respond(message, response_message_id, workspace)
debug_msg("Python - Responded to message: " + str(message) +
", response: " + str(response))
response_consumer[0] = response
except Exception as ex:
error_message = str(ex)
debug_msg(error_message, exc_info=True)
# Message format: Error message, pretty traceback, number of traceback elements (frames), list of frames
# where each frame is of the form: filename, line number, name, line.
error_pretty_traceback = traceback.format_exc()
error_traceback_frames = traceback.extract_tb(sys.exc_info()[2])
error_payload = PayloadEncoder().put_string(
error_message).put_string(error_pretty_traceback).put_int(
len(error_traceback_frames))
for frame in reversed(error_traceback_frames):
error_payload.put_string(frame[0]).put_int(
frame[1]).put_string(frame[2]).put_string(frame[3])
error_payload = error_payload.payload
# Inform Java that handling the request did not work.
error_response = AbstractRequestHandler._create_response(
message,
response_message_id,
success=False,
response_payload=error_payload)
response_consumer[0] = error_response
result_consumer(
None
) # We are done after the response (either success or failure) is sent.
return True
def _handle_custom_message(self, message, response_message_id_supplier,
response_consumer, result_consumer, workspace):
response_message_id = response_message_id_supplier()
try:
debug_msg("Python - Respond to message: " + str(message))
response = self._respond(message, response_message_id, workspace)
debug_msg("Python - Responded to message: " + str(message) +
", response: " + str(response))
response_consumer[0] = response
except Exception as ex:
error_message = str(ex)
debug_msg(error_message, exc_info=True)
error_traceback = traceback.format_exc()
error_payload = PayloadEncoder().put_string(
error_message).put_string(error_traceback).payload
# Inform Java that handling the request did not work.
error_response = AbstractRequestHandler._create_response(
message,
response_message_id,
success=False,
response_payload=error_payload)
response_consumer[0] = error_response
result_consumer(
None
) # We are done after the response (either success or failure) is sent.
return True
def handle(self, message, message_handlers, message_id_supplier,
result_consumer, workspace):
message_type = message.get_header_field(
AbstractTaskHandler.FIELD_KEY_MESSAGE_TYPE)
debug_msg("Python - Handle task, message: " + str(message))
if message_type == AbstractTaskHandler.MESSAGE_TYPE_SUCCESS:
result = self._handle_success_message(message)
debug_msg("Python - Handled task, message: " + str(message) +
", result: " + str(result))
result_consumer(result)
elif message_type == AbstractTaskHandler.MESSAGE_TYPE_FAILURE:
self._handle_failure_message(message)
else:
message_to_send = [None]
if self._handle_custom_message(message, message_id_supplier,
message_to_send, result_consumer,
workspace):
debug_msg("Python - Handled task, message: " + str(message) +
", follow-up: " + str(message_to_send[0]))
return message_to_send[0]
else:
if not message_handlers.handle(message):
raise RuntimeError("Message '" + str(message) +
"' could not be handled.")
return None