def with_idle_state_retention_time(self, min_time, max_time): """ Specifies a minimum and a maximum time interval for how long idle state, i.e., state which was not updated, will be retained. State will never be cleared until it was idle for less than the minimum time and will never be kept if it was idle for more than the maximum time. When new data arrives for previously cleaned-up state, the new data will be handled as if it was the first data. This can result in previous results being overwritten. Set to ``datetime.timedelta()``(zero) to never clean-up the state. .. note:: Cleaning up state requires additional bookkeeping which becomes less expensive for larger differences of minTime and maxTime. The difference between minTime and maxTime must be at least ``datetime.timedelta(minutes=5)``(5 minutes). :param min_time: The minimum time interval for which idle state is retained. Set to ``datetime.timedelta()``(zero) to never clean-up the state. :param max_time: The maximum time interval for which idle state is retained. Must be at least 5 minutes greater than minTime. Set to ``datetime.timedelta()``(zero) to never clean-up the state. :return: :class:`StreamQueryConfig` """ # type: (timedelta, timedelta) -> StreamQueryConfig j_time_class = get_gateway().jvm.org.apache.flink.api.common.time.Time j_min_time = j_time_class.milliseconds(long(round(min_time.total_seconds() * 1000))) j_max_time = j_time_class.milliseconds(long(round(max_time.total_seconds() * 1000))) self._j_stream_query_config = \ self._j_stream_query_config.withIdleStateRetentionTime(j_min_time, j_max_time) return self
def with_idle_state_retention_time(self, min_time, max_time): """ Specifies a minimum and a maximum time interval for how long idle state, i.e., state which was not updated, will be retained. State will never be cleared until it was idle for less than the minimum time and will never be kept if it was idle for more than the maximum time. When new data arrives for previously cleaned-up state, the new data will be handled as if it was the first data. This can result in previous results being overwritten. Set to ``datetime.timedelta()``(zero) to never clean-up the state. .. note:: Cleaning up state requires additional bookkeeping which becomes less expensive for larger differences of minTime and maxTime. The difference between minTime and maxTime must be at least ``datetime.timedelta(minutes=5)``(5 minutes). :param min_time: The minimum time interval for which idle state is retained. Set to ``datetime.timedelta()``(zero) to never clean-up the state. :param max_time: The maximum time interval for which idle state is retained. Must be at least 5 minutes greater than minTime. Set to ``datetime.timedelta()``(zero) to never clean-up the state. :return: :class:`StreamQueryConfig` """ # type: (timedelta, timedelta) -> StreamQueryConfig j_time_class = get_gateway().jvm.org.apache.flink.api.common.time.Time j_min_time = j_time_class.milliseconds( long(round(min_time.total_seconds() * 1000))) j_max_time = j_time_class.milliseconds( long(round(max_time.total_seconds() * 1000))) self._j_stream_query_config = \ self._j_stream_query_config.withIdleStateRetentionTime(j_min_time, j_max_time) return self
def set_idle_state_retention(self, duration: datetime.timedelta): """ Specifies a retention time interval for how long idle state, i.e., state which was not updated, will be retained. State will never be cleared until it was idle for less than the duration and will never be kept if it was idle for more than the 1.5 x duration. When new data arrives for previously cleaned-up state, the new data will be handled as if it was the first data. This can result in previous results being overwritten. Set to 0 (zero) to never clean-up the state. Example: :: >>> table_config.set_idle_state_retention(datetime.timedelta(days=1)) .. note:: Cleaning up state requires additional bookkeeping which becomes less expensive for larger differences of minTime and maxTime. The difference between minTime and maxTime must be at least 5 minutes. :param duration: The retention time interval for which idle state is retained. Set to 0 (zero) to never clean-up the state. """ j_duration_class = get_gateway().jvm.java.time.Duration j_duration = j_duration_class.ofMillis(long(round(duration.total_seconds() * 1000))) self._j_table_config.setIdleStateRetention(j_duration)
def testProtocolReceive(self): p = start_echo_server_process() time.sleep(1) try: testSocket = get_socket() testSocket.sendall('yo\n'.encode('utf-8')) testSocket.sendall('yro0\n'.encode('utf-8')) testSocket.sendall('yo\n'.encode('utf-8')) testSocket.sendall('ysHello World\n'.encode('utf-8')) # No extra echange (method3) because it is already cached. testSocket.sendall('yi123\n'.encode('utf-8')) testSocket.sendall('yd1.25\n'.encode('utf-8')) testSocket.sendall('yo\n'.encode('utf-8')) testSocket.sendall('yn\n'.encode('utf-8')) testSocket.sendall('yo\n'.encode('utf-8')) testSocket.sendall('ybTrue\n'.encode('utf-8')) testSocket.sendall('yo\n'.encode('utf-8')) testSocket.sendall('yL123\n'.encode('utf-8')) testSocket.close() time.sleep(1) self.gateway = JavaGateway(auto_field=True) ex = self.gateway.getNewExample() self.assertEqual('Hello World', ex.method3(1, True)) self.assertEqual(123, ex.method3()) self.assertAlmostEqual(1.25, ex.method3()) self.assertTrue(ex.method2() is None) self.assertTrue(ex.method4()) self.assertEqual(long(123), ex.method8()) self.gateway.shutdown() except Exception: print_exc() self.fail('Problem occurred') p.join()
def set_idle_state_retention_time(self, min_time, max_time): """ Specifies a minimum and a maximum time interval for how long idle state, i.e., state which was not updated, will be retained. State will never be cleared until it was idle for less than the minimum time and will never be kept if it was idle for more than the maximum time. When new data arrives for previously cleaned-up state, the new data will be handled as if it was the first data. This can result in previous results being overwritten. Set to 0 (zero) to never clean-up the state. Example: :: >>> table_config = TableConfig() \\ ... .set_idle_state_retention_time(datetime.timedelta(days=1), ... datetime.timedelta(days=3)) .. note:: Cleaning up state requires additional bookkeeping which becomes less expensive for larger differences of minTime and maxTime. The difference between minTime and maxTime must be at least 5 minutes. Method set_idle_state_retention_time is deprecated now. The suggested way to set idle state retention time is :func:`~pyflink.table.TableConfig.set_idle_state_retention` Currently, setting max_time will not work and the max_time is directly derived from the min_time as 1.5 x min_time. :param min_time: The minimum time interval for which idle state is retained. Set to 0 (zero) to never clean-up the state. :type min_time: datetime.timedelta :param max_time: The maximum time interval for which idle state is retained. Must be at least 5 minutes greater than minTime. Set to 0 (zero) to never clean-up the state. :type max_time: datetime.timedelta """ j_time_class = get_gateway().jvm.org.apache.flink.api.common.time.Time j_min_time = j_time_class.milliseconds( long(round(min_time.total_seconds() * 1000))) j_max_time = j_time_class.milliseconds( long(round(max_time.total_seconds() * 1000))) self._j_table_config.setIdleStateRetentionTime(j_min_time, j_max_time)
def testAutoField(self): self.gateway = JavaGateway(auto_field=True) ex = self.gateway.getNewExample() self.assertEqual(ex.field10, 10) self.assertEqual(ex.field11, long(11)) sb = ex.field20 sb.append('Hello') self.assertEqual('Hello', sb.toString()) self.assertTrue(ex.field21 == None)
def testAutoField(self): self.gateway = JavaGateway(gateway_parameters=GatewayParameters(auto_field=True)) ex = self.gateway.getNewExample() self.assertEqual(ex.field10, 10) self.assertEqual(ex.field11, long(11)) sb = ex.field20 sb.append("Hello") self.assertEqual("Hello", sb.toString()) self.assertTrue(ex.field21 is None)
def testAutoField(self): self.gateway = JavaGateway(gateway_parameters=GatewayParameters( auto_field=True)) ex = self.gateway.getNewExample() self.assertEqual(ex.field10, 10) self.assertEqual(ex.field11, long(11)) sb = ex.field20 sb.append("Hello") self.assertEqual("Hello", sb.toString()) self.assertTrue(ex.field21 is None)
def testLongInt(self): ex = self.gateway.getNewExample() self.assertEqual(1, ex.method7(1234)) self.assertEqual(4, ex.method7(2147483648)) self.assertEqual(4, ex.method7(long(2147483648))) self.assertEqual(long(4), ex.method8(3)) self.assertEqual(4, ex.method8(3)) self.assertEqual(long(4), ex.method8(long(3))) self.assertEqual(long(4), ex.method9(long(3)))
def testLongInt(self): ex = self.gateway.getNewExample() self.assertEqual(1, ex.method7(1234)) self.assertEqual(4, ex.method7(2147483648)) self.assertEqual(4, ex.method7(-2147483649)) self.assertEqual(4, ex.method7(long(2147483648))) self.assertEqual(long(4), ex.method8(3)) self.assertEqual(4, ex.method8(3)) self.assertEqual(long(4), ex.method8(long(3))) self.assertEqual(long(4), ex.method9(long(3))) try: ex.method8(3000000000000000000000000000000000000) self.fail("Should not be able to convert overflowing long") except Py4JError: self.assertTrue(True) # Check that the connection is not broken (refs #265) self.assertEqual(4, ex.method8(3))
def testProtocolReceive(self): p = start_echo_server_process() try: testSocket = get_socket() testSocket.sendall("!yo\n".encode("utf-8")) testSocket.sendall("!yro0\n".encode("utf-8")) testSocket.sendall("!yo\n".encode("utf-8")) testSocket.sendall("!ysHello World\n".encode("utf-8")) # No extra echange (method3) because it is already cached. testSocket.sendall("!yi123\n".encode("utf-8")) testSocket.sendall("!yd1.25\n".encode("utf-8")) testSocket.sendall("!yo\n".encode("utf-8")) testSocket.sendall("!yn\n".encode("utf-8")) testSocket.sendall("!yo\n".encode("utf-8")) testSocket.sendall("!ybTrue\n".encode("utf-8")) testSocket.sendall("!yo\n".encode("utf-8")) testSocket.sendall("!yL123\n".encode("utf-8")) testSocket.sendall("!ydinf\n".encode("utf-8")) testSocket.close() sleep() self.gateway = JavaGateway( gateway_parameters=GatewayParameters(auto_field=True)) ex = self.gateway.getNewExample() self.assertEqual("Hello World", ex.method3(1, True)) self.assertEqual(123, ex.method3()) self.assertAlmostEqual(1.25, ex.method3()) self.assertTrue(ex.method2() is None) self.assertTrue(ex.method4()) self.assertEqual(long(123), ex.method8()) self.assertEqual(float("inf"), ex.method8()) self.gateway.shutdown() except Exception: print_exc() self.fail("Problem occurred") p.join()
def testProtocolReceive(self): p = start_echo_server_process() try: testSocket = get_socket() testSocket.sendall("!yo\n".encode("utf-8")) testSocket.sendall("!yro0\n".encode("utf-8")) testSocket.sendall("!yo\n".encode("utf-8")) testSocket.sendall("!ysHello World\n".encode("utf-8")) # No extra echange (method3) because it is already cached. testSocket.sendall("!yi123\n".encode("utf-8")) testSocket.sendall("!yd1.25\n".encode("utf-8")) testSocket.sendall("!yo\n".encode("utf-8")) testSocket.sendall("!yn\n".encode("utf-8")) testSocket.sendall("!yo\n".encode("utf-8")) testSocket.sendall("!ybTrue\n".encode("utf-8")) testSocket.sendall("!yo\n".encode("utf-8")) testSocket.sendall("!yL123\n".encode("utf-8")) testSocket.sendall("!ydinf\n".encode("utf-8")) testSocket.close() sleep() self.gateway = JavaGateway(gateway_parameters=GatewayParameters( auto_field=True)) ex = self.gateway.getNewExample() self.assertEqual("Hello World", ex.method3(1, True)) self.assertEqual(123, ex.method3()) self.assertAlmostEqual(1.25, ex.method3()) self.assertTrue(ex.method2() is None) self.assertTrue(ex.method4()) self.assertEqual(long(123), ex.method8()) self.assertEqual(float("inf"), ex.method8()) self.gateway.shutdown() except Exception: print_exc() self.fail("Problem occurred") p.join()
AUTH_COMMAND_NAME = "A" CALL_PROXY_COMMAND_NAME = "c" GARBAGE_COLLECT_PROXY_COMMAND_NAME = "g" SERVER_STATUS_COMMAND_NAME = "s" KILL_THREAD_COMMAND_NAME = "k" # Dir subcommands DIR_FIELDS_SUBCOMMAND_NAME = "f\n" DIR_METHODS_SUBCOMMAND_NAME = "m\n" DIR_STATIC_SUBCOMMAND_NAME = "s\n" DIR_JVMVIEW_SUBCOMMAND_NAME = "v\n" OUTPUT_CONVERTER = { NULL_TYPE: (lambda x, y: None), BOOLEAN_TYPE: (lambda value, y: value.lower() == "true"), LONG_TYPE: (lambda value, y: long(value)), DECIMAL_TYPE: (lambda value, y: Decimal(value)), INTEGER_TYPE: (lambda value, y: int(value)), BYTES_TYPE: (lambda value, y: decode_bytearray(value)), DOUBLE_TYPE: (lambda value, y: float(value)), STRING_TYPE: (lambda value, y: unescape_new_line(value)), } INPUT_CONVERTER = [] # ERRORS ERROR_ON_SEND = "on_send" ERROR_ON_RECEIVE = "on_receive" def escape_new_line(original):
ERROR_RETURN_MESSAGE = RETURN_MESSAGE + ERROR + "\n" SUCCESS_RETURN_MESSAGE = RETURN_MESSAGE + SUCCESS + "\n" CALL_PROXY_COMMAND_NAME = "c" GARBAGE_COLLECT_PROXY_COMMAND_NAME = "g" # Dir subcommands DIR_FIELDS_SUBCOMMAND_NAME = "f\n" DIR_METHODS_SUBCOMMAND_NAME = "m\n" DIR_STATIC_SUBCOMMAND_NAME = "s\n" DIR_JVMVIEW_SUBCOMMAND_NAME = "v\n" OUTPUT_CONVERTER = { NULL_TYPE: (lambda x, y: None), BOOLEAN_TYPE: (lambda value, y: value.lower() == "true"), LONG_TYPE: (lambda value, y: long(value)), DECIMAL_TYPE: (lambda value, y: Decimal(value)), INTEGER_TYPE: (lambda value, y: int(value)), BYTES_TYPE: (lambda value, y: decode_bytearray(value)), DOUBLE_TYPE: (lambda value, y: float(value)), STRING_TYPE: (lambda value, y: unescape_new_line(value)), } INPUT_CONVERTER = [] # ERRORS ERROR_ON_SEND = "on_send" ERROR_ON_RECEIVE = "on_receive" def escape_new_line(original):
def normalize_date(spark: SparkSession, value: datetime, granularity: str) -> datetime: instant: long = long(value.timestamp() * 1000) normalized: long = spark.sparkContext._jvm.com.rovio.ingest.util \ .NormalizeTimeColumnUDF.normalize(instant, granularity) return datetime.fromtimestamp(normalized / 1000)