def grpc_table_op(self, table: Table, op: TableOp):
try:
result_id = self.session.make_ticket()
if table:
table_reference = table_pb2.TableReference(ticket=table.ticket)
else:
table_reference = None
stub_func = op.__class__.get_stub_func(self._grpc_table_stub)
response = stub_func(op.make_grpc_request(
result_id=result_id, source_id=table_reference),
metadata=self.session.grpc_metadata)
if response.success:
return Table(self.session,
ticket=response.result_id.ticket,
schema_header=response.schema_header,
size=response.size,
is_static=response.is_static)
else:
raise DHError(
f"Server error received for {op.__class__.__name__}: {response.error_info}"
)
except Exception as e:
raise DHError(
f"failed to finish {op.__class__.__name__} operation") from e
def import_table(self, data: pyarrow.Table):
try:
options = paflight.FlightCallOptions(
headers=self.session.grpc_metadata)
if not isinstance(data, (pa.Table, pa.RecordBatch)):
raise DHError(
"source data must be either a pa table or RecordBatch.")
ticket = self.session.get_ticket()
dh_fields = []
for f in data.schema:
dh_fields.append(
pa.field(name=f.name,
type=f.type,
metadata=_map_arrow_type(f.type)))
dh_schema = pa.schema(dh_fields)
writer, reader = self._flight_client.do_put(
pa.flight.FlightDescriptor.for_path("export", str(ticket)),
dh_schema,
options=options)
writer.write_table(data)
writer.close()
_ = reader.read()
flight_ticket = self.session.make_ticket(ticket)
return Table(self.session,
ticket=flight_ticket,
size=data.num_rows,
schema=dh_schema)
except Exception as e:
raise DHError(
"failed to create a Deephaven table from Arrow data.") from e
def bind_table(self, table, variable_name):
if not table or not variable_name:
raise DHError("invalid table and/or variable_name values.")
try:
response = self._grpc_console_stub.BindTableToVariable(
console_pb2.BindTableToVariableRequest(console_id=self.console_id,
table_id=table.ticket,
variable_name=variable_name),
metadata=self.session.grpc_metadata)
except Exception as e:
raise DHError("failed to bind a table to a variable on the server.") from e
def close(self):
try:
self._grpc_session_stub.CloseSession(
session_pb2.HandshakeRequest(auth_protocol=0, payload=self.session.session_token),
metadata=self.session.grpc_metadata)
except Exception as e:
raise DHError("failed to close the session.") from e
def sync_fields(self, repeating: bool):
""" Check for fields that have been added/deleted by other sessions and add them to the local list
This will start a new background thread when `repeating=True`.
Args:
repeating (bool): Continue to check in the background for new/updated tables
Raises:
DHError
"""
with self._r_lock:
if self._list_fields is not None:
return
self._list_fields = self.app_service.list_fields()
self._parse_fields_change(next(self._list_fields))
if repeating:
self._field_update_thread = threading.Thread(target=self._update_fields)
self._field_update_thread.daemon = True
self._field_update_thread.start()
else:
if not self._list_fields.cancel():
raise DHError("could not cancel ListFields subscription")
self._list_fields = None
def get_ticket(self):
with self._r_lock:
self._last_ticket += 1
if self._last_ticket == 2**31 - 1:
raise DHError("fatal error: out of free internal ticket")
return self._last_ticket
def refresh_token(self):
try:
response = self._grpc_session_stub.RefreshSessionToken(
session_pb2.HandshakeRequest(auth_protocol=0, payload=self.session.session_token),
metadata=self.session.grpc_metadata)
return response.session_token, response.token_expiration_delay_millis
except Exception as e:
raise DHError("failed to refresh session token.") from e
def list_fields(self):
try:
fields = self._grpc_app_stub.ListFields(
application_pb2.ListFieldsRequest(),
metadata=self.session.grpc_metadata)
return fields
except Exception as e:
raise DHError("failed to list fields.") from e
def snapshot_table(self, table: Table):
try:
options = paflight.FlightCallOptions(
headers=self.session.grpc_metadata)
flight_ticket = paflight.Ticket(table.ticket.ticket)
reader = self._flight_client.do_get(flight_ticket, options=options)
return reader.read_all()
except Exception as e:
raise DHError("failed to take a snapshot of the table.") from e
def __init__(self, session=None, ticket=None, schema_header=b'', size=None, is_static=None, schema=None):
if not session or not session.is_alive:
raise DHError("Must be associated with a active session")
self.session = session
self.ticket = ticket
self.schema = schema
self.is_static = is_static
self.size = size
if not schema:
self._parse_schema(schema_header)
def connect(self):
grpc_channel = grpc.insecure_channel(":".join([self.session.host, str(self.session.port)]))
self._grpc_session_stub = session_pb2_grpc.SessionServiceStub(grpc_channel)
try:
response = self._grpc_session_stub.NewSession(
session_pb2.HandshakeRequest(auth_protocol=1, payload=b'hello pydeephaven'))
return grpc_channel, response.session_token, response.token_expiration_delay_millis
except Exception as e:
grpc_channel.close()
raise DHError("failed to connect to the server.") from e
def batch(self, ops):
batch_op = BatchOpAssembler(self.session, table_ops=ops).build_batch()
try:
response = self._grpc_table_stub.Batch(
table_pb2.BatchTableRequest(ops=batch_op),
metadata=self.session.grpc_metadata)
exported_tables = []
for exported in response:
if not exported.success:
raise DHError(exported.error_info)
if exported.result_id.WhichOneof("ref") == "ticket":
exported_tables.append(Table(self.session, ticket=exported.result_id.ticket,
schema_header=exported.schema_header,
size=exported.size,
is_static=exported.is_static))
return exported_tables[-1]
except Exception as e:
raise DHError("failed to finish the table batch operation.") from e
def open_table(self, name):
self.start_console()
try:
result_id = self.session.make_ticket()
response = self._grpc_console_stub.FetchTable(
console_pb2.FetchTableRequest(console_id=self.console_id,
table_id=result_id,
table_name=name),
metadata=self.session.grpc_metadata)
if response.success:
return Table(self.session, ticket=response.result_id.ticket,
schema_header=response.schema_header,
size=response.size,
is_static=response.is_static)
else:
raise DHError("error open a table: " + response.error_info)
except Exception as e:
raise DHError("failed to open a table.") from e
def run_script(self, server_script):
self.start_console()
try:
response = self._grpc_console_stub.ExecuteCommand(
console_pb2.ExecuteCommandRequest(console_id=self.console_id,
code=server_script),
metadata=self.session.grpc_metadata)
return response
except Exception as e:
raise DHError("failed to execute a command in the console.") from e
def start_console(self):
if self.console_id:
return
try:
result_id = self.session.make_ticket()
response = self._grpc_console_stub.StartConsole(
console_pb2.StartConsoleRequest(result_id=result_id, session_type='python'),
metadata=self.session.grpc_metadata)
self.console_id = response.result_id
except Exception as e:
raise DHError("failed to start a console.") from e
def __init__(self, host: str = None, port: int = None, never_timeout: bool = True, session_type: str = 'python', sync_fields: int = NO_SYNC):
""" Initialize a Session object that connects to the Deephaven server
Args:
host (str): the host name or IP address of the remote machine, default is 'localhost'
port (int): the port number that Deephaven server is listening on, default is 10000
never_timeout (bool, optional): never allow the session to timeout, default is True
session_type (str, optional): the Deephaven session type. Defaults to 'python'
sync_fields (int, optional): equivalent to calling `Session.sync_fields()` (see below), default is NO_SYNC
Sync Options:
session.NO_SYNC: does not check for existing tables on the server
session.SYNC_ONCE: equivalent to `Session.sync_fields(repeating=False)`
session.SYNC_REPEATED: equivalent to `Session.sync_fields(repeating=True)`
Raises:
DHError
"""
self._r_lock = threading.RLock()
self._last_ticket = 0
self._ticket_bitarray = BitArray(1024)
self.host = host
if not host:
self.host = os.environ.get("DH_HOST", "localhost")
self.port = port
if not port:
self.port = int(os.environ.get("DH_PORT", 10000))
if sync_fields not in (NO_SYNC, SYNC_ONCE, SYNC_REPEATED):
raise DHError("invalid sync_fields setting")
self.is_connected = False
self.session_token = None
self.grpc_channel = None
self._session_service = None
self._table_service = None
self._grpc_barrage_stub = None
self._console_service = None
self._flight_service = None
self._app_service = None
self._never_timeout = never_timeout
self._keep_alive_timer = None
self._session_type = session_type
self._sync_fields = sync_fields
self._list_fields = None
self._field_update_thread = None
self._fields = {}
self._connect()
def _map_arrow_type(arrow_type):
arrow_to_dh = {
pa.null(): '',
pa.bool_(): '',
pa.int8(): 'byte',
pa.int16(): 'short',
pa.int32(): 'int',
pa.int64(): 'long',
pa.uint8(): '',
pa.uint16(): 'char',
pa.uint32(): '',
pa.uint64(): '',
pa.float16(): '',
pa.float32(): 'float',
pa.float64(): 'double',
pa.time32('s'): '',
pa.time32('ms'): '',
pa.time64('us'): '',
pa.time64('ns'): 'io.deephaven.time.DateTime',
pa.timestamp('us', tz=None): '',
pa.timestamp('ns', tz=None): '',
pa.date32(): 'java.time.LocalDate',
pa.date64(): 'java.time.LocalDate',
pa.binary(): '',
pa.string(): 'java.lang.String',
pa.utf8(): 'java.lang.String',
pa.large_binary(): '',
pa.large_string(): '',
pa.large_utf8(): '',
# decimal128(int precision, int scale=0)
# list_(value_type, int list_size=-1)
# large_list(value_type)
# map_(key_type, item_type[, keys_sorted])
# struct(fields)
# dictionary(index_type, value_type, …)
# field(name, type, bool nullable = True[, metadata])
# schema(fields[, metadata])
# from_numpy_dtype(dtype)
}
dh_type = arrow_to_dh.get(arrow_type)
if not dh_type:
# if this is a case of timestamp with tz specified
if isinstance(arrow_type, pa.TimestampType):
dh_type = "io.deephaven.time.DateTime"
if not dh_type:
raise DHError(f'unsupported arrow data type : {arrow_type}')
return {"deephaven:type": dh_type}
def open_table(self, name: str) -> Table:
""" Open a table in the global scope with the given name on the server.
Args:
name (str): the name of the table
Returns:
a Table object
Raises:
DHError
"""
with self._r_lock:
if name not in self.tables:
raise DHError(f"no table by the name {name}")
table_op = FetchTableOp()
return self.table_service.grpc_table_op(self._fields[('scope', name)][1], table_op)
def release(self, ticket):
try:
self._grpc_session_stub.Release(session_pb2.ReleaseRequest(id=ticket), metadata=self.session.grpc_metadata)
except Exception as e:
raise DHError("failed to release a ticket.") from e
def __init__(self, session, table):
self.session = session
if not self.session or not table:
raise DHError("invalid session or table value.")
self._ops = [NoneOp(table=table)]