def process():
try:
input_data = ser.load_stream(infile)
code = CodeCache.get(command)
if is_interactive:
global data_manager
global context
data_manager = PythonContext(context_id, input_data, conf)
context = data_manager
global globals_namespace
exec(code, globals_namespace, globals_namespace)
else:
data_manager = PythonContext(context_id, input_data, conf)
n_local = {
"data_manager": data_manager,
"context": data_manager
}
exec(code, n_local, n_local)
out_iter = data_manager.output()
write_int(SpecialLengths.START_ARROW_STREAM, outfile)
out_ser.dump_stream(out_iter, outfile)
finally:
try:
import shutil
shutil.rmtree(context_id)
except:
pass
try:
if hasattr(out_iter, 'close'):
out_iter.close()
except:
pass
try:
del data_manager
except:
pass
def process():
input_data = ser.load_stream(infile)
code = compile(command, '<string>', 'exec')
if is_interactive:
global data_manager
global context
data_manager = PythonContext(input_data, conf)
context = data_manager
global globals_namespace
exec (code, globals_namespace, globals_namespace)
else:
data_manager = PythonContext(input_data, conf)
n_local = {"data_manager": data_manager, "context": data_manager}
g_local = {}
exec (code, g_local, n_local)
out_iter = data_manager.output()
try:
write_int(SpecialLengths.START_ARROW_STREAM, outfile)
out_ser.dump_stream(out_iter, outfile)
finally:
if hasattr(out_iter, 'close'):
out_iter.close()
def serve(self, data):
from pyjava.api.mlsql import PythonContext
if not self.is_bind:
raise SocketNotBindException(
"Please invoke server.bind() before invoke server.serve")
conn, addr = self.socket.accept()
sockfile = conn.makefile("rwb", int(
os.environ.get("BUFFER_SIZE", 65536)))
infile = sockfile # os.fdopen(os.dup(conn.fileno()), "rb", 65536)
out = sockfile # os.fdopen(os.dup(conn.fileno()), "wb", 65536)
try:
write_int(SpecialLengths.START_ARROW_STREAM, out)
out_data = ([df[name] for name in df] for df in
PythonContext.build_chunk_result(data, 1024))
self.out_ser.dump_stream(out_data, out)
write_int(SpecialLengths.END_OF_DATA_SECTION, out)
write_int(SpecialLengths.END_OF_STREAM, out)
out.flush()
if self.is_dev:
print("all data in ray task have been consumed.")
read_int(infile)
except Exception:
try:
write_int(SpecialLengths.ARROW_STREAM_CRASH, out)
ex = traceback.format_exc()
print(ex)
write_int(SpecialLengths.PYTHON_EXCEPTION_THROWN, out)
write_with_length(ex.encode("utf-8"), out)
out.flush()
read_int(infile)
except IOError:
# JVM close the socket
pass
except Exception:
# Write the error to stderr if it happened while serializing
print("Py worker failed with exception:")
print(traceback.format_exc())
pass
conn.close()
from pyjava.api.mlsql import PythonContext
import pyarrow as pa
import pandas as pd
df = pd.read_csv("user_visit")
conf = {}
input_data = [pa.RecordBatch.from_pandas(df)]
data_manager = PythonContext(1 ,input_data, conf)
############
def process(_data_manager):
pv = 0
uv = 0
lastUrl = None
lastUid = None
output = False
for item in _data_manager.fetch_once_as_rows():
urlCol = item["url"]
uidCol = item["uid"]
if lastUrl is None:
pv = 1
uv = 1
lastUrl = urlCol
lastUid = uidCol
elif lastUrl == urlCol:
pv = pv + 1
if lastUid != uidCol:
def show_plt(plt: Any, context: PythonContext) -> NoReturn:
content = Utils.gen_img(plt)
context.build_result([{"content": content, "mime": "image"}])
from __future__ import absolute_import
import ray
from pyjava.api.mlsql import PythonContext
from pyjava.api import is_in_mlsql
ray.shutdown()
data = [{'id': 9, 'content': '1', 'label': 0.0}]
wow = PythonContext.build_chunk_result(data, 1024)
# items = ([df[name] for name in df] for df in wow)
for item in wow:
print(item)