def with_idle_state_retention_time(self, min_time, max_time):
"""
Specifies a minimum and a maximum time interval for how long idle state, i.e., state which
was not updated, will be retained.
State will never be cleared until it was idle for less than the minimum time and will never
be kept if it was idle for more than the maximum time.
When new data arrives for previously cleaned-up state, the new data will be handled as if it
was the first data. This can result in previous results being overwritten.
Set to ``datetime.timedelta()``(zero) to never clean-up the state.
.. note::
Cleaning up state requires additional bookkeeping which becomes less expensive for
larger differences of minTime and maxTime. The difference between minTime and maxTime
must be at least ``datetime.timedelta(minutes=5)``(5 minutes).
:param min_time: The minimum time interval for which idle state is retained. Set to
``datetime.timedelta()``(zero) to never clean-up the state.
:param max_time: The maximum time interval for which idle state is retained. Must be at
least 5 minutes greater than minTime. Set to
``datetime.timedelta()``(zero) to never clean-up the state.
:return: :class:`StreamQueryConfig`
"""
# type: (timedelta, timedelta) -> StreamQueryConfig
j_time_class = get_gateway().jvm.org.apache.flink.api.common.time.Time
j_min_time = j_time_class.milliseconds(long(round(min_time.total_seconds() * 1000)))
j_max_time = j_time_class.milliseconds(long(round(max_time.total_seconds() * 1000)))
self._j_stream_query_config = \
self._j_stream_query_config.withIdleStateRetentionTime(j_min_time, j_max_time)
return self
def with_idle_state_retention_time(self, min_time, max_time):
"""
Specifies a minimum and a maximum time interval for how long idle state, i.e., state which
was not updated, will be retained.
State will never be cleared until it was idle for less than the minimum time and will never
be kept if it was idle for more than the maximum time.
When new data arrives for previously cleaned-up state, the new data will be handled as if it
was the first data. This can result in previous results being overwritten.
Set to ``datetime.timedelta()``(zero) to never clean-up the state.
.. note::
Cleaning up state requires additional bookkeeping which becomes less expensive for
larger differences of minTime and maxTime. The difference between minTime and maxTime
must be at least ``datetime.timedelta(minutes=5)``(5 minutes).
:param min_time: The minimum time interval for which idle state is retained. Set to
``datetime.timedelta()``(zero) to never clean-up the state.
:param max_time: The maximum time interval for which idle state is retained. Must be at
least 5 minutes greater than minTime. Set to
``datetime.timedelta()``(zero) to never clean-up the state.
:return: :class:`StreamQueryConfig`
"""
# type: (timedelta, timedelta) -> StreamQueryConfig
j_time_class = get_gateway().jvm.org.apache.flink.api.common.time.Time
j_min_time = j_time_class.milliseconds(
long(round(min_time.total_seconds() * 1000)))
j_max_time = j_time_class.milliseconds(
long(round(max_time.total_seconds() * 1000)))
self._j_stream_query_config = \
self._j_stream_query_config.withIdleStateRetentionTime(j_min_time, j_max_time)
return self
def set_idle_state_retention(self, duration: datetime.timedelta):
"""
Specifies a retention time interval for how long idle state, i.e., state which
was not updated, will be retained.
State will never be cleared until it was idle for less than the duration and will never
be kept if it was idle for more than the 1.5 x duration.
When new data arrives for previously cleaned-up state, the new data will be handled as if it
was the first data. This can result in previous results being overwritten.
Set to 0 (zero) to never clean-up the state.
Example:
::
>>> table_config.set_idle_state_retention(datetime.timedelta(days=1))
.. note::
Cleaning up state requires additional bookkeeping which becomes less expensive for
larger differences of minTime and maxTime. The difference between minTime and maxTime
must be at least 5 minutes.
:param duration: The retention time interval for which idle state is retained. Set to
0 (zero) to never clean-up the state.
"""
j_duration_class = get_gateway().jvm.java.time.Duration
j_duration = j_duration_class.ofMillis(long(round(duration.total_seconds() * 1000)))
self._j_table_config.setIdleStateRetention(j_duration)
def testProtocolReceive(self):
p = start_echo_server_process()
time.sleep(1)
try:
testSocket = get_socket()
testSocket.sendall('yo\n'.encode('utf-8'))
testSocket.sendall('yro0\n'.encode('utf-8'))
testSocket.sendall('yo\n'.encode('utf-8'))
testSocket.sendall('ysHello World\n'.encode('utf-8'))
# No extra echange (method3) because it is already cached.
testSocket.sendall('yi123\n'.encode('utf-8'))
testSocket.sendall('yd1.25\n'.encode('utf-8'))
testSocket.sendall('yo\n'.encode('utf-8'))
testSocket.sendall('yn\n'.encode('utf-8'))
testSocket.sendall('yo\n'.encode('utf-8'))
testSocket.sendall('ybTrue\n'.encode('utf-8'))
testSocket.sendall('yo\n'.encode('utf-8'))
testSocket.sendall('yL123\n'.encode('utf-8'))
testSocket.close()
time.sleep(1)
self.gateway = JavaGateway(auto_field=True)
ex = self.gateway.getNewExample()
self.assertEqual('Hello World', ex.method3(1, True))
self.assertEqual(123, ex.method3())
self.assertAlmostEqual(1.25, ex.method3())
self.assertTrue(ex.method2() is None)
self.assertTrue(ex.method4())
self.assertEqual(long(123), ex.method8())
self.gateway.shutdown()
except Exception:
print_exc()
self.fail('Problem occurred')
p.join()
def testProtocolReceive(self):
p = start_echo_server_process()
time.sleep(1)
try:
testSocket = get_socket()
testSocket.sendall('yo\n'.encode('utf-8'))
testSocket.sendall('yro0\n'.encode('utf-8'))
testSocket.sendall('yo\n'.encode('utf-8'))
testSocket.sendall('ysHello World\n'.encode('utf-8'))
# No extra echange (method3) because it is already cached.
testSocket.sendall('yi123\n'.encode('utf-8'))
testSocket.sendall('yd1.25\n'.encode('utf-8'))
testSocket.sendall('yo\n'.encode('utf-8'))
testSocket.sendall('yn\n'.encode('utf-8'))
testSocket.sendall('yo\n'.encode('utf-8'))
testSocket.sendall('ybTrue\n'.encode('utf-8'))
testSocket.sendall('yo\n'.encode('utf-8'))
testSocket.sendall('yL123\n'.encode('utf-8'))
testSocket.close()
time.sleep(1)
self.gateway = JavaGateway(auto_field=True)
ex = self.gateway.getNewExample()
self.assertEqual('Hello World', ex.method3(1, True))
self.assertEqual(123, ex.method3())
self.assertAlmostEqual(1.25, ex.method3())
self.assertTrue(ex.method2() is None)
self.assertTrue(ex.method4())
self.assertEqual(long(123), ex.method8())
self.gateway.shutdown()
except Exception:
print_exc()
self.fail('Problem occurred')
p.join()
def set_idle_state_retention_time(self, min_time, max_time):
"""
Specifies a minimum and a maximum time interval for how long idle state, i.e., state which
was not updated, will be retained.
State will never be cleared until it was idle for less than the minimum time and will never
be kept if it was idle for more than the maximum time.
When new data arrives for previously cleaned-up state, the new data will be handled as if it
was the first data. This can result in previous results being overwritten.
Set to 0 (zero) to never clean-up the state.
Example:
::
>>> table_config = TableConfig() \\
... .set_idle_state_retention_time(datetime.timedelta(days=1),
... datetime.timedelta(days=3))
.. note::
Cleaning up state requires additional bookkeeping which becomes less expensive for
larger differences of minTime and maxTime. The difference between minTime and maxTime
must be at least 5 minutes.
Method set_idle_state_retention_time is deprecated now. The suggested way to set idle
state retention time is :func:`~pyflink.table.TableConfig.set_idle_state_retention`
Currently, setting max_time will not work and the max_time is directly derived from the
min_time as 1.5 x min_time.
:param min_time: The minimum time interval for which idle state is retained. Set to
0 (zero) to never clean-up the state.
:type min_time: datetime.timedelta
:param max_time: The maximum time interval for which idle state is retained. Must be at
least 5 minutes greater than minTime. Set to
0 (zero) to never clean-up the state.
:type max_time: datetime.timedelta
"""
j_time_class = get_gateway().jvm.org.apache.flink.api.common.time.Time
j_min_time = j_time_class.milliseconds(
long(round(min_time.total_seconds() * 1000)))
j_max_time = j_time_class.milliseconds(
long(round(max_time.total_seconds() * 1000)))
self._j_table_config.setIdleStateRetentionTime(j_min_time, j_max_time)
def testAutoField(self):
self.gateway = JavaGateway(auto_field=True)
ex = self.gateway.getNewExample()
self.assertEqual(ex.field10, 10)
self.assertEqual(ex.field11, long(11))
sb = ex.field20
sb.append('Hello')
self.assertEqual('Hello', sb.toString())
self.assertTrue(ex.field21 == None)
def testAutoField(self):
self.gateway = JavaGateway(gateway_parameters=GatewayParameters(auto_field=True))
ex = self.gateway.getNewExample()
self.assertEqual(ex.field10, 10)
self.assertEqual(ex.field11, long(11))
sb = ex.field20
sb.append("Hello")
self.assertEqual("Hello", sb.toString())
self.assertTrue(ex.field21 is None)
def testAutoField(self):
self.gateway = JavaGateway(auto_field=True)
ex = self.gateway.getNewExample()
self.assertEqual(ex.field10, 10)
self.assertEqual(ex.field11, long(11))
sb = ex.field20
sb.append('Hello')
self.assertEqual('Hello', sb.toString())
self.assertTrue(ex.field21 == None)
def testAutoField(self):
self.gateway = JavaGateway(gateway_parameters=GatewayParameters(
auto_field=True))
ex = self.gateway.getNewExample()
self.assertEqual(ex.field10, 10)
self.assertEqual(ex.field11, long(11))
sb = ex.field20
sb.append("Hello")
self.assertEqual("Hello", sb.toString())
self.assertTrue(ex.field21 is None)
def testLongInt(self):
ex = self.gateway.getNewExample()
self.assertEqual(1, ex.method7(1234))
self.assertEqual(4, ex.method7(2147483648))
self.assertEqual(4, ex.method7(long(2147483648)))
self.assertEqual(long(4), ex.method8(3))
self.assertEqual(4, ex.method8(3))
self.assertEqual(long(4), ex.method8(long(3)))
self.assertEqual(long(4), ex.method9(long(3)))
def testLongInt(self):
ex = self.gateway.getNewExample()
self.assertEqual(1, ex.method7(1234))
self.assertEqual(4, ex.method7(2147483648))
self.assertEqual(4, ex.method7(long(2147483648)))
self.assertEqual(long(4), ex.method8(3))
self.assertEqual(4, ex.method8(3))
self.assertEqual(long(4), ex.method8(long(3)))
self.assertEqual(long(4), ex.method9(long(3)))
def testLongInt(self):
ex = self.gateway.getNewExample()
self.assertEqual(1, ex.method7(1234))
self.assertEqual(4, ex.method7(2147483648))
self.assertEqual(4, ex.method7(-2147483649))
self.assertEqual(4, ex.method7(long(2147483648)))
self.assertEqual(long(4), ex.method8(3))
self.assertEqual(4, ex.method8(3))
self.assertEqual(long(4), ex.method8(long(3)))
self.assertEqual(long(4), ex.method9(long(3)))
try:
ex.method8(3000000000000000000000000000000000000)
self.fail("Should not be able to convert overflowing long")
except Py4JError:
self.assertTrue(True)
# Check that the connection is not broken (refs #265)
self.assertEqual(4, ex.method8(3))
def testLongInt(self):
ex = self.gateway.getNewExample()
self.assertEqual(1, ex.method7(1234))
self.assertEqual(4, ex.method7(2147483648))
self.assertEqual(4, ex.method7(-2147483649))
self.assertEqual(4, ex.method7(long(2147483648)))
self.assertEqual(long(4), ex.method8(3))
self.assertEqual(4, ex.method8(3))
self.assertEqual(long(4), ex.method8(long(3)))
self.assertEqual(long(4), ex.method9(long(3)))
try:
ex.method8(3000000000000000000000000000000000000)
self.fail("Should not be able to convert overflowing long")
except Py4JError:
self.assertTrue(True)
# Check that the connection is not broken (refs #265)
self.assertEqual(4, ex.method8(3))
def testProtocolReceive(self):
p = start_echo_server_process()
try:
testSocket = get_socket()
testSocket.sendall("!yo\n".encode("utf-8"))
testSocket.sendall("!yro0\n".encode("utf-8"))
testSocket.sendall("!yo\n".encode("utf-8"))
testSocket.sendall("!ysHello World\n".encode("utf-8"))
# No extra echange (method3) because it is already cached.
testSocket.sendall("!yi123\n".encode("utf-8"))
testSocket.sendall("!yd1.25\n".encode("utf-8"))
testSocket.sendall("!yo\n".encode("utf-8"))
testSocket.sendall("!yn\n".encode("utf-8"))
testSocket.sendall("!yo\n".encode("utf-8"))
testSocket.sendall("!ybTrue\n".encode("utf-8"))
testSocket.sendall("!yo\n".encode("utf-8"))
testSocket.sendall("!yL123\n".encode("utf-8"))
testSocket.sendall("!ydinf\n".encode("utf-8"))
testSocket.close()
sleep()
self.gateway = JavaGateway(
gateway_parameters=GatewayParameters(auto_field=True))
ex = self.gateway.getNewExample()
self.assertEqual("Hello World", ex.method3(1, True))
self.assertEqual(123, ex.method3())
self.assertAlmostEqual(1.25, ex.method3())
self.assertTrue(ex.method2() is None)
self.assertTrue(ex.method4())
self.assertEqual(long(123), ex.method8())
self.assertEqual(float("inf"), ex.method8())
self.gateway.shutdown()
except Exception:
print_exc()
self.fail("Problem occurred")
p.join()
def testProtocolReceive(self):
p = start_echo_server_process()
try:
testSocket = get_socket()
testSocket.sendall("!yo\n".encode("utf-8"))
testSocket.sendall("!yro0\n".encode("utf-8"))
testSocket.sendall("!yo\n".encode("utf-8"))
testSocket.sendall("!ysHello World\n".encode("utf-8"))
# No extra echange (method3) because it is already cached.
testSocket.sendall("!yi123\n".encode("utf-8"))
testSocket.sendall("!yd1.25\n".encode("utf-8"))
testSocket.sendall("!yo\n".encode("utf-8"))
testSocket.sendall("!yn\n".encode("utf-8"))
testSocket.sendall("!yo\n".encode("utf-8"))
testSocket.sendall("!ybTrue\n".encode("utf-8"))
testSocket.sendall("!yo\n".encode("utf-8"))
testSocket.sendall("!yL123\n".encode("utf-8"))
testSocket.sendall("!ydinf\n".encode("utf-8"))
testSocket.close()
sleep()
self.gateway = JavaGateway(gateway_parameters=GatewayParameters(
auto_field=True))
ex = self.gateway.getNewExample()
self.assertEqual("Hello World", ex.method3(1, True))
self.assertEqual(123, ex.method3())
self.assertAlmostEqual(1.25, ex.method3())
self.assertTrue(ex.method2() is None)
self.assertTrue(ex.method4())
self.assertEqual(long(123), ex.method8())
self.assertEqual(float("inf"), ex.method8())
self.gateway.shutdown()
except Exception:
print_exc()
self.fail("Problem occurred")
p.join()
AUTH_COMMAND_NAME = "A"
CALL_PROXY_COMMAND_NAME = "c"
GARBAGE_COLLECT_PROXY_COMMAND_NAME = "g"
SERVER_STATUS_COMMAND_NAME = "s"
KILL_THREAD_COMMAND_NAME = "k"
# Dir subcommands
DIR_FIELDS_SUBCOMMAND_NAME = "f\n"
DIR_METHODS_SUBCOMMAND_NAME = "m\n"
DIR_STATIC_SUBCOMMAND_NAME = "s\n"
DIR_JVMVIEW_SUBCOMMAND_NAME = "v\n"
OUTPUT_CONVERTER = {
NULL_TYPE: (lambda x, y: None),
BOOLEAN_TYPE: (lambda value, y: value.lower() == "true"),
LONG_TYPE: (lambda value, y: long(value)),
DECIMAL_TYPE: (lambda value, y: Decimal(value)),
INTEGER_TYPE: (lambda value, y: int(value)),
BYTES_TYPE: (lambda value, y: decode_bytearray(value)),
DOUBLE_TYPE: (lambda value, y: float(value)),
STRING_TYPE: (lambda value, y: unescape_new_line(value)),
}
INPUT_CONVERTER = []
# ERRORS
ERROR_ON_SEND = "on_send"
ERROR_ON_RECEIVE = "on_receive"
def escape_new_line(original):
ERROR_RETURN_MESSAGE = RETURN_MESSAGE + ERROR + "\n"
SUCCESS_RETURN_MESSAGE = RETURN_MESSAGE + SUCCESS + "\n"
CALL_PROXY_COMMAND_NAME = "c"
GARBAGE_COLLECT_PROXY_COMMAND_NAME = "g"
# Dir subcommands
DIR_FIELDS_SUBCOMMAND_NAME = "f\n"
DIR_METHODS_SUBCOMMAND_NAME = "m\n"
DIR_STATIC_SUBCOMMAND_NAME = "s\n"
DIR_JVMVIEW_SUBCOMMAND_NAME = "v\n"
OUTPUT_CONVERTER = {
NULL_TYPE: (lambda x, y: None),
BOOLEAN_TYPE: (lambda value, y: value.lower() == "true"),
LONG_TYPE: (lambda value, y: long(value)),
DECIMAL_TYPE: (lambda value, y: Decimal(value)),
INTEGER_TYPE: (lambda value, y: int(value)),
BYTES_TYPE: (lambda value, y: decode_bytearray(value)),
DOUBLE_TYPE: (lambda value, y: float(value)),
STRING_TYPE: (lambda value, y: unescape_new_line(value)),
}
INPUT_CONVERTER = []
# ERRORS
ERROR_ON_SEND = "on_send"
ERROR_ON_RECEIVE = "on_receive"
def escape_new_line(original):
def normalize_date(spark: SparkSession, value: datetime,
granularity: str) -> datetime:
instant: long = long(value.timestamp() * 1000)
normalized: long = spark.sparkContext._jvm.com.rovio.ingest.util \
.NormalizeTimeColumnUDF.normalize(instant, granularity)
return datetime.fromtimestamp(normalized / 1000)
