def add_table(self, table: Table, col: str) -> Table: """Registers a table for replaying and returns the associated replay table. Args: table (Table): the table to be replayed col (str): column in the table containing timestamps Returns: a replay Table Raises: DHError """ try: replay_table = Table(j_table=self._j_replayer.replay(table.j_table, col)) return replay_table except Exception as e: raise DHError(e, "failed to add a historical table.") from e
def query_update_performance(eval_number: int) -> Table: """ Takes in a query evaluation number and returns a view for that query's update performance data. You can obtain query evaluation numbers, which uniquely identify a query and its subqueries, via the performance data tables obtained from calling query_performance_log() or query_operation_performance_log() Args: eval_number (int): the evaluation number Returns: a Table of query update performance data Raises: DHError """ try: return Table(j_table=_JPerformanceQueries.queryUpdatePerformance(eval_number)) except Exception as e: raise DHError(e, "failed to obtain the query update performance data.") from e
def merge_sorted(tables: List[Table], order_by: str) -> Table: """Combines two or more tables into one sorted, aggregate table. This essentially stacks the tables one on top of the other and sorts the result. Null tables are ignored. mergeSorted is more efficient than using merge followed by sort. Args: tables (List[Table]): the source tables order_by (str): the name of the key column Returns: a Table Raises: DHError """ try: return Table(j_table=_JTableTools.mergeSorted(order_by, *[t.j_table for t in tables])) except Exception as e: raise DHError(e, "merge sorted operation failed.") from e
def query_update_performance_map(eval_number: int) -> Dict[str, Table]: """ Creates multiple tables with performance data for a given query identified by an evaluation number. The tables are returned in a map with the following String keys: 'QueryUpdatePerformance', 'UpdateWorst', 'WorstInterval', 'UpdateMostRecent', 'UpdateAggregate', 'UpdateSummaryStats'. Args: eval_number (int): the evaluation number Returns: a dict Raises: DHError """ try: d = j_map_to_dict(_JPerformanceQueries.queryUpdatePerformanceMap(eval_number)) for k in d.keys(): d[k] = Table(j_table=d[k]) return d except Exception as e: raise DHError(e, "failed to obtain the query update perf map.") from e
def consume( kafka_config: Dict, cdc_spec: CDCSpec, partitions: List[int] = None, stream_table: bool = False, cols_to_drop: List[str] = None, ) -> Table: """ Consume from a Change Data Capture (CDC) Kafka stream (as, eg, produced by Debezium), tracking the underlying database table to a Deephaven table. Args: kafka_config (Dict): configuration for the associated kafka consumer and also the resulting table. Passed to the org.apache.kafka.clients.consumer.KafkaConsumer constructor; pass any KafkaConsumer specific desired configuration here. Note this should include the relevant property for a schema server URL where the key and/or value Avro necessary schemas are stored. cdc_spec (CDCSpec): a CDCSpec obtained from calling either the cdc_long_spec or the cdc_short_spec function partitions (List[int]: a list of integer partition numbers, default is None indicating all partitions stream_table (bool): if true, produce a streaming table of changed rows keeping the CDC 'op' column indicating the type of column change; if false, return a Deephaven ticking table that tracks the underlying database table through the CDC Stream. cols_to_drop (list[str]): a list of column names to omit from the resulting DHC table. Note that only columns not included in the primary key for the table can be dropped at this stage; you can chain a drop column operation after this call if you need to do this. Returns: a Deephaven live table that will update based on the CDC messages consumed for the given topic Raises: DHError """ try: partitions = j_partitions(partitions) kafka_config = j_properties(kafka_config) return Table(j_table=_JCdcTools.consumeToTable( kafka_config, cdc_spec.j_object, partitions, stream_table, cols_to_drop)) except Exception as e: raise DHError(e, "failed to consume a CDC stream.") from e
def query_operation_performance(eval_number: int) -> Table: """ Takes in a query evaluation number and returns a view for that query's individual operation's performance data. You can obtain query evaluation numbers, which uniquely identify a query and its subqueries, via the performance data tables obtained from calling query_performance_log() or query_operation_performance_log() The query operation performance log contains data on how long each individual operation of a query (where(), update(), naturalJoin(), etc., as well as internal functions) takes to execute, and the change in resource consumption while each was executing. Args: eval_number (int): the evaluation number Returns: a table of query operation performance data Raises: DHError """ try: return Table(j_table=_JPerformanceQueries.queryOperationPerformance(eval_number)) except Exception as e: raise DHError(e, "failed to obtain the query operation performance data.") from e
def time_window(table: Table, ts_col: str, window: int, bool_col: str) -> Table: """Creates a new table by applying a time window to the source table and adding a new Boolean column. The value of the new Boolean column is set to false when the timestamp column value is older than the window from now or true otherwise. If the timestamp column value is null, the Boolean column value will be null as well. The result table ticks whenever the source table ticks, or modifies a row when it passes out of the window. Args: table (Table): the source table ts_col (str): the timestamp column name window (int): the size of the window in nanoseconds bool_col (str): the name of the new Boolean column. Returns: a new Table Raises: DHError """ try: return Table(j_table=_JWindowCheck.addTimeWindow(table.j_table, ts_col, window, bool_col)) except Exception as e: raise DHError(e, "failed to create a time window table.") from e
def consume( kafka_config: Dict, topic: str, partitions: List[int] = None, offsets: Dict[int, int] = None, key_spec: KeyValueSpec = None, value_spec: KeyValueSpec = None, table_type: TableType = TableType.stream(), ) -> Table: """Consume from Kafka to a Deephaven table. Args: kafka_config (Dict): configuration for the associated Kafka consumer and also the resulting table. Once the table-specific properties are stripped, the remaining one is used to call the constructor of org.apache.kafka.clients.consumer.KafkaConsumer; pass any KafkaConsumer specific desired configuration here topic (str): the Kafka topic name partitions (List[int]) : a list of integer partition numbers, default is None which means all partitions offsets (Dict[int, int]) : a mapping between partition numbers and offset numbers, and can be one of the predefined ALL_PARTITIONS_SEEK_TO_BEGINNING, ALL_PARTITIONS_SEEK_TO_END or ALL_PARTITIONS_DONT_SEEK. The default is None which works the same as ALL_PARTITIONS_DONT_SEEK. The offset numbers may be one of the predefined SEEK_TO_BEGINNING, SEEK_TO_END, or DONT_SEEK. key_spec (KeyValueSpec): specifies how to map the Key field in Kafka messages to Deephaven column(s). It can be the result of calling one of the functions: simple_spec(),avro_spec() or json_spec() in this module, or the predefined KeyValueSpec.IGNORE or KeyValueSpec.FROM_PROPERTIES. The default is None which works the same as KeyValueSpec.FROM_PROPERTIES, in which case, the kafka_config param should include values for dictionary keys 'deephaven.key.column.name' and 'deephaven.key.column.type', for the single resulting column name and type value_spec (KeyValueSpec): specifies how to map the Value field in Kafka messages to Deephaven column(s). It can be the result of calling one of the functions: simple_spec(),avro_spec() or json_spec() in this module, or the predefined KeyValueSpec.IGNORE or KeyValueSpec.FROM_PROPERTIES. The default is None which works the same as KeyValueSpec.FROM_PROPERTIES, in which case, the kafka_config param should include values for dictionary keys 'deephaven.key.column.name' and 'deephaven.key.column.type', for the single resulting column name and type table_type (TableType): a TableType enum, default is TableType.stream() Returns: a Deephaven live table that will update based on Kafka messages consumed for the given topic Raises: DHError """ try: partitions = j_partitions(partitions) if offsets is None or offsets == ALL_PARTITIONS_DONT_SEEK: offsets = _ALL_PARTITIONS_DONT_SEEK elif offsets == ALL_PARTITIONS_SEEK_TO_BEGINNING: offsets = _ALL_PARTITIONS_SEEK_TO_BEGINNING elif offsets == ALL_PARTITIONS_SEEK_TO_END: offsets = _ALL_PARTITIONS_SEEK_TO_END else: partitions_array = jpy.array("int", list(offsets.keys())) offsets_array = jpy.array("long", list(offsets.values())) offsets = _JKafkaTools.partitionToOffsetFromParallelArrays( partitions_array, offsets_array) key_spec = KeyValueSpec.FROM_PROPERTIES if key_spec is None else key_spec value_spec = KeyValueSpec.FROM_PROPERTIES if value_spec is None else value_spec if key_spec is KeyValueSpec.IGNORE and value_spec is KeyValueSpec.IGNORE: raise ValueError( "at least one argument for 'key' or 'value' must be different from KeyValueSpec.IGNORE" ) kafka_config = j_properties(kafka_config) return Table(j_table=_JKafkaTools.consumeToTable( kafka_config, topic, partitions, offsets, key_spec.j_object, value_spec.j_object, table_type.j_object, )) except Exception as e: raise DHError(e, "failed to consume a Kafka stream.") from e
def read( path: str, header: Dict[str, dht.DType] = None, headless: bool = False, skip_rows: int = 0, num_rows: int = MAX_LONG, ignore_empty_lines: bool = False, allow_missing_columns: bool = False, ignore_excess_columns: bool = False, delimiter: str = ",", quote: str = '"', ignore_surrounding_spaces: bool = True, trim: bool = False, ) -> Table: """Read the CSV data specified by the path parameter as a table. Args: path (str): a file path or a URL string header (Dict[str, DType]): a dict to define the table columns with key being the name, value being the data type skip_rows (long): number of data rows to skip before processing data. This is useful when you want to parse data in chunks. Defaults to 0 num_rows (long): max number of rows to process. This is useful when you want to parse data in chunks. Defaults to {@link Long#MAX_VALUE} allow_missing_columns (bool): whether the library should allow missing columns in the input. If this flag is set, then rows that are too short (that have fewer columns than the header row) will be interpreted as if the missing columns contained the empty string. Defaults to false. ignore_excess_columns (bool): whether the library should allow excess columns in the input. If this flag is set, then rows that are too long (that have more columns than the header row) will have those excess columns dropped. Defaults to false. headless (bool): indicates if the CSV data is headless, default is False delimiter (str): the delimiter used by the CSV, default is the comma quote (str): the quote character for the CSV, default is double quote ignore_surrounding_spaces (bool): Indicates whether surrounding white space should be ignored for unquoted text fields, default is True trim (bool): indicates whether to trim white space inside a quoted string, default is False Returns: a table Raises: DHError """ try: csv_specs_builder = _JCsvTools.builder() if header: csv_specs_builder.headers(_JArrays.asList(list(header.keys()))) parser_map = { dht.bool_: _JParsers.BOOLEAN, dht.byte: _JParsers.BYTE, dht.char: _JParsers.CHAR, dht.short: _JParsers.SHORT, dht.int_: _JParsers.INT, dht.long: _JParsers.LONG, dht.float_: _JParsers.FLOAT_FAST, dht.double: _JParsers.DOUBLE, dht.string: _JParsers.STRING, dht.DateTime: _JParsers.DATETIME, } for column_name, column_type in header.items(): csv_specs_builder.putParserForName(column_name, parser_map[column_type]) csv_specs = (csv_specs_builder.hasHeaderRow(not headless).skipRows( skip_rows).numRows(num_rows).ignoreEmptyLines( ignore_empty_lines).allowMissingColumns(allow_missing_columns). ignoreExcessColumns(ignore_excess_columns).delimiter( ord(delimiter)).quote( ord(quote)).ignoreSurroundingSpaces( ignore_surrounding_spaces).trim(trim).build()) j_table = _JCsvTools.readCsv(path, csv_specs) return Table(j_table=j_table) except Exception as e: raise DHError(e, "read csv failed") from e