def process(): try: input_data = ser.load_stream(infile) code = CodeCache.get(command) if is_interactive: global data_manager global context data_manager = PythonContext(context_id, input_data, conf) context = data_manager global globals_namespace exec(code, globals_namespace, globals_namespace) else: data_manager = PythonContext(context_id, input_data, conf) n_local = { "data_manager": data_manager, "context": data_manager } exec(code, n_local, n_local) out_iter = data_manager.output() write_int(SpecialLengths.START_ARROW_STREAM, outfile) out_ser.dump_stream(out_iter, outfile) finally: try: import shutil shutil.rmtree(context_id) except: pass try: if hasattr(out_iter, 'close'): out_iter.close() except: pass try: del data_manager except: pass
def process(): input_data = ser.load_stream(infile) code = compile(command, '<string>', 'exec') if is_interactive: global data_manager global context data_manager = PythonContext(input_data, conf) context = data_manager global globals_namespace exec (code, globals_namespace, globals_namespace) else: data_manager = PythonContext(input_data, conf) n_local = {"data_manager": data_manager, "context": data_manager} g_local = {} exec (code, g_local, n_local) out_iter = data_manager.output() try: write_int(SpecialLengths.START_ARROW_STREAM, outfile) out_ser.dump_stream(out_iter, outfile) finally: if hasattr(out_iter, 'close'): out_iter.close()
def serve(self, data): from pyjava.api.mlsql import PythonContext if not self.is_bind: raise SocketNotBindException( "Please invoke server.bind() before invoke server.serve") conn, addr = self.socket.accept() sockfile = conn.makefile("rwb", int( os.environ.get("BUFFER_SIZE", 65536))) infile = sockfile # os.fdopen(os.dup(conn.fileno()), "rb", 65536) out = sockfile # os.fdopen(os.dup(conn.fileno()), "wb", 65536) try: write_int(SpecialLengths.START_ARROW_STREAM, out) out_data = ([df[name] for name in df] for df in PythonContext.build_chunk_result(data, 1024)) self.out_ser.dump_stream(out_data, out) write_int(SpecialLengths.END_OF_DATA_SECTION, out) write_int(SpecialLengths.END_OF_STREAM, out) out.flush() if self.is_dev: print("all data in ray task have been consumed.") read_int(infile) except Exception: try: write_int(SpecialLengths.ARROW_STREAM_CRASH, out) ex = traceback.format_exc() print(ex) write_int(SpecialLengths.PYTHON_EXCEPTION_THROWN, out) write_with_length(ex.encode("utf-8"), out) out.flush() read_int(infile) except IOError: # JVM close the socket pass except Exception: # Write the error to stderr if it happened while serializing print("Py worker failed with exception:") print(traceback.format_exc()) pass conn.close()
from pyjava.api.mlsql import PythonContext import pyarrow as pa import pandas as pd df = pd.read_csv("user_visit") conf = {} input_data = [pa.RecordBatch.from_pandas(df)] data_manager = PythonContext(1 ,input_data, conf) ############ def process(_data_manager): pv = 0 uv = 0 lastUrl = None lastUid = None output = False for item in _data_manager.fetch_once_as_rows(): urlCol = item["url"] uidCol = item["uid"] if lastUrl is None: pv = 1 uv = 1 lastUrl = urlCol lastUid = uidCol elif lastUrl == urlCol: pv = pv + 1 if lastUid != uidCol:
def show_plt(plt: Any, context: PythonContext) -> NoReturn: content = Utils.gen_img(plt) context.build_result([{"content": content, "mime": "image"}])
from __future__ import absolute_import import ray from pyjava.api.mlsql import PythonContext from pyjava.api import is_in_mlsql ray.shutdown() data = [{'id': 9, 'content': '1', 'label': 0.0}] wow = PythonContext.build_chunk_result(data, 1024) # items = ([df[name] for name in df] for df in wow) for item in wow: print(item)