def test_timestamp_nanos(self):
Timestamp(1, 0)
Timestamp(1, 999999999)
self.assertRaises(ValueError, Timestamp, 1, -1)
self.assertRaises(ValueError, Timestamp, 1, -2)
self.assertRaises(ValueError, Timestamp, 1, 1000000000)
self.assertRaises(ValueError, Timestamp, 1, 5000000000)
def test_timestamp_event_time_attribute(self):
topo = Topology('test_timestamp_event_time_attribute')
ts1 = Timestamp(1608196, 235000000, 0)
ts2 = Timestamp(1608199, 876265298, 0)
ts_schema = StreamSchema('tuple<int64 num, timestamp ts>').as_tuple(
named=True)
s = topo.source([(1, ts1), (2, ts2)])
# map() named "event_time_source" transforms to structured schema
s = s.map(lambda x: x, schema=ts_schema, name='event_time_source')
# add event-time annotation for attribute ts to the "event_time_source"
s = s.set_event_time('ts')
# use SPL function getEventTime() to get the event time of the input tuple
# copies the event-time timestamp value to a new output attribute "eventtime"
f = op.Map('spl.relational::Functor',
s,
schema=StreamSchema('tuple<timestamp eventtime>').as_tuple(
named=True))
f.eventtime = f.output('getEventTime(event_time_source)')
# map to Python schema (prepare for content comparision)
as_ts = f.stream.map(lambda x: x.eventtime)
tester = Tester(topo)
tester.tuple_count(s, 2)
tester.contents(as_ts, [ts1, ts2])
tester.test(self.test_ctxtype, self.test_config)
def test_window_spl_aggregate(self):
topo = Topology('test_window_spl_aggregate')
ts1 = Timestamp(1608196, 235000000, 0)
ts2 = Timestamp(1608199, 876265298, 0)
ts3 = Timestamp(1608506, 123456078, 0)
ts4 = Timestamp(1608507, 654326980, 0)
ts_schema = StreamSchema('tuple<int64 num, timestamp ts>').as_tuple(
named=True)
s = topo.source([(1, ts1), (2, ts2), (3, ts3), (4, ts4)])
# map() named "event_time_source" transforms to structured schema
s = s.map(lambda x: x, schema=ts_schema, name='event_time_source')
# add event-time annotation for attribute ts to the "event_time_source"
s = s.set_event_time('ts')
agg_schema = StreamSchema(
'tuple<uint64 sum, rstring pane_timing>').as_tuple(named=True)
# timeInterval window
win = s.time_interval(interval_duration=5.0, discard_age=30.0)
agg = op.Map('spl.relational::Aggregate', win, schema=agg_schema)
agg.sum = agg.output('Sum((uint64)num)')
agg.pane_timing = agg.output('(rstring)paneTiming()')
#agg.stream.print()
result = agg.stream.map(lambda x: x.sum)
result_pane_timing = agg.stream.map(lambda x: x.pane_timing)
tester = Tester(topo)
tester.tuple_count(result, 2)
tester.contents(result, [3, 7])
tester.contents(result_pane_timing,
['paneOnComplete', 'paneOnComplete'])
tester.test(self.test_ctxtype, self.test_config)
def test_DatetimeToTimestamp(self):
dt = datetime.datetime.now()
ts = Timestamp.from_datetime(dt)
self.assertEqual(dt, ts.datetime())
self.assertEqual(0, ts.machine_id)
ts = Timestamp.from_datetime(dt, 892)
self.assertEqual(dt, ts.datetime())
self.assertEqual(892, ts.machine_id)
def test_DatetimeToTimestamp(self):
dt = datetime.datetime.now()
ts = Timestamp.from_datetime(dt)
self.assertEqual(dt, ts.datetime())
self.assertEqual(0, ts.machine_id)
ts = Timestamp.from_datetime(dt, 892)
self.assertEqual(dt, ts.datetime())
self.assertEqual(892, ts.machine_id)
def test_TimestampToDatetime(self):
# 2017-06-04 11:48:25.008880
ts = Timestamp(1496576905, 888000000, 0)
dt = ts.datetime()
self.assertIsInstance(dt, datetime.datetime)
self.assertIsNone(dt.tzinfo)
self.assertEqual(2017, dt.year)
self.assertEqual(6, dt.month)
self.assertEqual(4, dt.day)
self.assertEqual(11, dt.hour)
self.assertEqual(48, dt.minute)
self.assertEqual(25, dt.second)
def test_TimestampToDatetime(self):
# 2017-06-04 11:48:25.008880
ts = Timestamp(1496576905, 888000000, 0)
dt = ts.datetime()
self.assertIsInstance(dt, datetime.datetime)
self.assertIsNone(dt.tzinfo)
self.assertEqual(2017, dt.year)
self.assertEqual(6, dt.month)
self.assertEqual(4, dt.day)
self.assertEqual(11, dt.hour)
self.assertEqual(48, dt.minute)
self.assertEqual(25, dt.second)
def test_good(self):
for dt in SPL_TYPES:
if dt in GOOD_DATA:
data = GOOD_DATA[dt]
topo = Topology()
schema = StreamSchema('tuple<' + dt + ' a>')
s = topo.source(data)
c = s.map(lambda x : (x,), schema=schema)
#c.print(tag=dt)
if dt.startswith('float'):
expected = [{'a':float(d)} for d in data]
elif dt.startswith('int'):
expected = [{'a':int(d)} for d in data]
elif dt == 'decimal32':
ctx = decimal.Context(prec=7, rounding=decimal.ROUND_HALF_EVEN)
expected = [{'a':decimal.Decimal(str(d)).normalize(ctx)} for d in data]
elif dt == 'decimal64':
ctx = decimal.Context(prec=16, rounding=decimal.ROUND_HALF_EVEN)
expected = [{'a':decimal.Decimal(str(d)).normalize(ctx)} for d in data]
elif dt == 'decimal128':
ctx = decimal.Context(prec=34, rounding=decimal.ROUND_HALF_EVEN)
expected = [{'a':decimal.Decimal(str(d)).normalize(ctx)} for d in data]
elif dt.startswith('complex'):
expected = [{'a':complex(d)} for d in data]
elif dt == 'timestamp':
expected = [{'a': d if isinstance(d, Timestamp) else Timestamp.from_datetime(d)} for d in data]
tester = Tester(topo)
tester.tuple_count(c, len(data))
tester.contents(c, expected)
tester.test(self.test_ctxtype, self.test_config)
def __iter__(self):
rv = (
True,
23, -2525, 3252352, -2624565653,
72, 6873, 43665588, 357568872,
4367.34, -87657525334.22,
"⡍⠔⠙⠖ ⡊ ⠙⠕⠝⠰⠞ ⠍⠑⠁⠝ ⠞⠕ ⠎⠁⠹ ⠹⠁⠞ ⡊ ⠅⠝⠪⠂ ⠕⠋ ⠍⠹",
complex(-23.0, 325.38), complex(-35346.234, 952524.93),
decimal.Decimal("3.459876E72"),
decimal.Decimal("4.515716038731674E-307"),
decimal.Decimal("1.085059319410602846995696978141388E+5922"),
Timestamp(781959759, 9320, 76),
"binary blob 8322".encode('utf-8'),
["a", "Streams!", "2H₂ + O₂ ⇌ 2H₂O, R = 4.7 kΩ, ⌀ 200 mm"],
[345,-4578],
[9983, -4647787587, 0],
[87346],
[45433674, 41876984848],
[4.269986E+05, -8.072285E+02, -6.917091E-08, 7.735085E8],
[765.46477e19],
[True, False, 7],
{},
{},
{'abc':35320, 'многоязычных':-236325}
)
return itertools.repeat(rv, 1)
def test_good(self):
for dt in SPL_TYPES:
if dt in GOOD_DATA:
data = GOOD_DATA[dt]
topo = Topology()
schema = StreamSchema('tuple<' + dt + ' a>')
s = topo.source(data)
c = s.map(lambda x : (x,), schema=schema)
#c.print(tag=dt)
if dt.startswith('float'):
expected = [{'a':float(d)} for d in data]
elif dt.startswith('int'):
expected = [{'a':int(d)} for d in data]
elif dt == 'decimal32':
ctx = decimal.Context(prec=7, rounding=decimal.ROUND_HALF_EVEN)
expected = [{'a':decimal.Decimal(str(d)).normalize(ctx)} for d in data]
elif dt == 'decimal64':
ctx = decimal.Context(prec=16, rounding=decimal.ROUND_HALF_EVEN)
expected = [{'a':decimal.Decimal(str(d)).normalize(ctx)} for d in data]
elif dt == 'decimal128':
ctx = decimal.Context(prec=34, rounding=decimal.ROUND_HALF_EVEN)
expected = [{'a':decimal.Decimal(str(d)).normalize(ctx)} for d in data]
elif dt.startswith('complex'):
expected = [{'a':complex(d)} for d in data]
elif dt == 'timestamp':
expected = [{'a': d if isinstance(d, Timestamp) else Timestamp.from_datetime(d)} for d in data]
tester = Tester(topo)
tester.tuple_count(c, len(data))
tester.contents(c, expected)
tester.test(self.test_ctxtype, self.test_config)
def test_Timestamp(self):
s = random.randint(0, 999999999999)
ns = random.randint(0, 1000000000)
mid = random.randint(0, 200000)
ts = Timestamp(s, ns, mid)
self.assertEqual(s, ts.seconds)
self.assertEqual(ns, ts.nanoseconds)
self.assertEqual(mid, ts.machine_id)
self.assertEqual(Timestamp, type(ts))
self.assertTrue(isinstance(ts, tuple))
t = ts.tuple()
self.assertEqual(3, len(t))
self.assertEqual(s, t[0])
self.assertEqual(ns, t[1])
self.assertEqual(mid, t[2])
ts2 = Timestamp(ts.seconds, ts.nanoseconds, ts.machine_id)
self.assertEqual(ts, ts2)
now = time.time()
ts2 = Timestamp(now, 0)
self.assertEqual(int(now), ts2.seconds)
self.assertEqual(0, ts2.nanoseconds)
self.assertEqual(0, ts2.machine_id)
s = random.randint(0, 999999999999)
ns = random.randint(0, 1000000000)
ts = Timestamp(s, ns)
self.assertEqual(s, ts.seconds)
self.assertEqual(ns, ts.nanoseconds)
self.assertEqual(0, ts.machine_id)
ft = ts.time()
self.assertIsInstance(ft, float)
eft = s + (ns / 1000.0 / 1000.0 / 1000.0)
self.assertEqual(eft, ft)
tsft = Timestamp.from_time(23423.02, 93)
self.assertEqual(23423, tsft.seconds)
self.assertEqual(20*1000.0*1000.0, float(tsft.nanoseconds))
self.assertEqual(93, tsft.machine_id)
def test_timestamp(self):
ts_schema = StreamSchema('tuple<int32 a, timestamp ts>').as_tuple(
named=True)
ts1 = Timestamp(133001, 302245576, 56)
ts2s = Timestamp(23543463, 876265298, 32)
dt1 = ts2s.datetime()
ts2 = Timestamp.from_datetime(dt1)
self.assertEqual(ts2s.seconds, ts2s.seconds)
topo = Topology()
s = topo.source([(1, ts1), (2, dt1)])
s = s.map(lambda x: x, schema=ts_schema)
as_ts = s.map(lambda x: x.ts)
tester = Tester(topo)
tester.tuple_check(s, ts_check)
tester.tuple_count(s, 2)
tester.contents(as_ts, [ts1, ts2])
tester.test(self.test_ctxtype, self.test_config)
def test_timestamp(self):
ts_schema = StreamSchema('tuple<int32 a, timestamp ts>').as_tuple(named=True)
ts1 = Timestamp(133001, 302245576, 56)
ts2s = Timestamp(23543463, 876265298, 32)
dt1 = ts2s.datetime()
ts2 = Timestamp.from_datetime(dt1)
self.assertEqual(ts2s.seconds, ts2s.seconds);
topo = Topology()
s = topo.source([(1,ts1), (2,dt1)])
s = s.map(lambda x : x, schema=ts_schema)
as_ts = s.map(lambda x : x.ts.tuple())
s.print()
tester = Tester(topo)
tester.tuple_check(s, ts_check)
tester.tuple_count(s, 2)
tester.contents(as_ts, [ts1.tuple(), ts2.tuple()])
tester.test(self.test_ctxtype, self.test_config)
def test_Timestamp(self):
s = random.randint(0, 999999999999)
ns = random.randint(0, 1000000000)
mid = random.randint(0, 200000)
ts = Timestamp(s, ns, mid)
self.assertEqual(s, ts.seconds)
self.assertEqual(ns, ts.nanoseconds)
self.assertEqual(mid, ts.machine_id)
t = ts.tuple()
self.assertEqual(3, len(t))
self.assertEqual(s, t[0])
self.assertEqual(ns, t[1])
self.assertEqual(mid, t[2])
s = random.randint(0, 999999999999)
ns = random.randint(0, 1000000000)
ts = Timestamp(s, ns)
self.assertEqual(s, ts.seconds)
self.assertEqual(ns, ts.nanoseconds)
self.assertEqual(0, ts.machine_id)
ft = ts.time()
self.assertIsInstance(ft, float)
eft = s + (ns / 1000.0 / 1000.0 / 1000.0)
self.assertEqual(eft, ft)
def test_Timestamp(self):
s = random.randint(0, 999999999999)
ns = random.randint(0, 1000000000)
mid = random.randint(0, 200000)
ts = Timestamp(s, ns, mid)
self.assertEqual(s, ts.seconds)
self.assertEqual(ns, ts.nanoseconds)
self.assertEqual(mid, ts.machine_id)
self.assertEqual(Timestamp, type(ts))
self.assertTrue(isinstance(ts, tuple))
t = ts.tuple()
self.assertEqual(3, len(t))
self.assertEqual(s, t[0])
self.assertEqual(ns, t[1])
self.assertEqual(mid, t[2])
ts2 = Timestamp(ts.seconds, ts.nanoseconds, ts.machine_id)
self.assertEqual(ts, ts2)
now = time.time()
ts2 = Timestamp(now, 0)
self.assertEqual(int(now), ts2.seconds)
self.assertEqual(0, ts2.nanoseconds)
self.assertEqual(0, ts2.machine_id)
s = random.randint(0, 999999999999)
ns = random.randint(0, 1000000000)
ts = Timestamp(s, ns)
self.assertEqual(s, ts.seconds)
self.assertEqual(ns, ts.nanoseconds)
self.assertEqual(0, ts.machine_id)
ft = ts.time()
self.assertIsInstance(ft, float)
eft = s + (ns / 1000.0 / 1000.0 / 1000.0)
self.assertEqual(eft, ft)
tsft = Timestamp.from_time(23423.02, 93)
self.assertEqual(23423, tsft.seconds)
self.assertEqual(20 * 1000.0 * 1000.0, float(tsft.nanoseconds))
self.assertEqual(93, tsft.machine_id)
def test_DatetimeToTimestamp(self):
dt = datetime.datetime.now()
ts = Timestamp.from_datetime(dt)
self.assertEqual(dt, ts.datetime())
self.assertEqual(0, ts.machine_id)
ts = Timestamp.from_datetime(dt, 892)
self.assertEqual(dt, ts.datetime())
self.assertEqual(892, ts.machine_id)
dt = ts.datetime()
self.assertIs(dt, ts.datetime())
# Check Timestamp is duck-typed as a datetime
self.assertEqual(dt.year, ts.year)
self.assertEqual(dt.month, ts.month)
self.assertEqual(dt.day, ts.day)
self.assertEqual(dt.hour, ts.hour)
self.assertEqual(dt.minute, ts.minute)
self.assertEqual(dt.second, ts.second)
self.assertEqual(dt.microsecond, ts.microsecond)
self.assertEqual(dt.tzinfo, ts.tzinfo)
self.assertEqual(dt.ctime(), ts.ctime())
def __iter__(self):
rv = (True, 23, -2525, 3252352, -2624565653, 72, 6873, 43665588,
357568872, 4367.34, -87657525334.22,
"⡍⠔⠙⠖ ⡊ ⠙⠕⠝⠰⠞ ⠍⠑⠁⠝ ⠞⠕ ⠎⠁⠹ ⠹⠁⠞ ⡊ ⠅⠝⠪⠂ ⠕⠋ ⠍⠹",
complex(-23.0, 325.38), complex(-35346.234, 952524.93),
Timestamp(781959759, 9320, 76),
["a", "Streams!", "2H₂ + O₂ ⇌ 2H₂O, R = 4.7 kΩ, ⌀ 200 mm"
], [345, -4578], [9983, -4647787587,
0], [87346], [45433674, 41876984848],
[4.269986E+05, -8.072285E+02, -6.917091E-08,
7.735085E8], [765.46477e19], [True, False, 7], {}, {}, {
'abc': 35320,
'многоязычных': -236325
})
return itertools.repeat(rv, 1)
def test_window_python_aggregate(self):
topo = Topology('test_window_python_aggregate')
ts1 = Timestamp(1608196, 235000000, 0)
ts_schema = StreamSchema('tuple<int64 num, timestamp ts>').as_tuple(
named=True)
s = topo.source([(1, ts1)])
# map() named "event_time_source" transforms to structured schema
s = s.map(lambda x: x, schema=ts_schema, name='event_time_source')
# add event-time annotation for attribute ts to the "event_time_source"
s = s.set_event_time('ts')
win = s.time_interval(interval_duration=10.0)
# expect TypeError: Time-Interval window is not supported
self.assertRaises(TypeError, win.aggregate, lambda t: sum(t.num))
'timestamp'
}
GOOD_DATA = {
'float32': [7.5, 3, 0, False],
'float64': [10.5, -2, 0, True],
'int8': [23.5, -7, 0, 127, -128, False],
'int16': [43.5, -7, 0, 32767, -32768, False],
'int32': [9.5, -7, 0, 2147483647, -2147483648, False],
'int64': [-83.5, -7, 0, 9223372036854775807, -9223372036854775808, False],
'decimal32': [-83.5, -7, 0, '4.33', decimal.Decimal('17.832')],
'decimal64': [-993.335, -8, 0, '933.4543', decimal.Decimal('4932.3221')],
'decimal128': [-83993.7883, -9, 0, '9355.332222', decimal.Decimal('5345.79745902883')],
'complex32': [complex(8.0, -32.0), 0, 10.5, 93],
'complex64': [complex(27.0, -8.0), 0, -83.5, 134],
'timestamp': [Timestamp.now(7), datetime.datetime.today()]
}
if np is not None:
GOOD_DATA['int8'].extend([np.int8(93), np.int8(-47)])
GOOD_DATA['int8'].extend([np.int8(93), np.int8(-47)])
GOOD_DATA['int16'].extend([np.int16(95), np.int16(-4232)])
GOOD_DATA['int32'].extend([np.int32(21031), np.int32(-54232)])
GOOD_DATA['int64'].extend([np.int64(23214234), np.int32(-4575775)])
GOOD_DATA['float32'].extend([np.float32(8.9)])
GOOD_DATA['float64'].extend([np.float64(92142.999)])
class TestConversion(unittest.TestCase):
""" Test conversions of Python values to SPL attributes/types.
"""
_multiprocess_can_split_ = True
def test_timestamp_now(self):
now = time.time()
ts = Timestamp.now()
self.assertTrue(ts.time() >= now)
def test_timestamp_now(self):
now = time.time()
ts = Timestamp.now()
self.assertTrue(ts.time() >= now)
def test_timestamp_dill(self):
ts = Timestamp(4, 5, 6)
tsp = dill.loads(dill.dumps(ts))
self.assertEqual(ts, tsp)
def test_timestamp_pickle(self):
ts = Timestamp(1, 2, 3)
tsp = pickle.loads(pickle.dumps(ts))
self.assertEqual(ts, tsp)
'float32': [7.5, 3, 0, False],
'float64': [10.5, -2, 0, True],
'int8': [23.5, -7, 0, 127, -128, False],
'int16': [43.5, -7, 0, 32767, -32768, False],
'int32': [9.5, -7, 0, 2147483647, -2147483648, False],
'int64': [-83.5, -7, 0, 9223372036854775807, -9223372036854775808, False],
'decimal32': [-83.5, -7, 0, '4.33',
decimal.Decimal('17.832')],
'decimal64': [-993.335, -8, 0, '933.4543',
decimal.Decimal('4932.3221')],
'decimal128':
[-83993.7883, -9, 0, '9355.332222',
decimal.Decimal('5345.79745902883')],
'complex32': [complex(8.0, -32.0), 0, 10.5, 93],
'complex64': [complex(27.0, -8.0), 0, -83.5, 134],
'timestamp': [Timestamp.now(7),
datetime.datetime.today()]
}
if np is not None:
GOOD_DATA['int8'].extend([np.int8(93), np.int8(-47)])
GOOD_DATA['int8'].extend([np.int8(93), np.int8(-47)])
GOOD_DATA['int16'].extend([np.int16(95), np.int16(-4232)])
GOOD_DATA['int32'].extend([np.int32(21031), np.int32(-54232)])
GOOD_DATA['int64'].extend([np.int64(23214234), np.int32(-4575775)])
GOOD_DATA['float32'].extend([np.float32(8.9)])
GOOD_DATA['float64'].extend([np.float64(92142.999)])
class TestConversion(unittest.TestCase):
""" Test conversions of Python values to SPL attributes/types.
'timestamp'
}
GOOD_DATA = {
'float32': [7.5, 3, 0, False],
'float64': [10.5, -2, 0, True],
'int8': [23.5, -7, 0, 127, -128, False],
'int16': [43.5, -7, 0, 32767, -32768, False],
'int32': [9.5, -7, 0, 2147483647, -2147483648, False],
'int64': [-83.5, -7, 0, 9223372036854775807, -9223372036854775808, False],
'decimal32': [-83.5, -7, 0, '4.33', decimal.Decimal('17.832')],
'decimal64': [-993.335, -8, 0, '933.4543', decimal.Decimal('4932.3221')],
'decimal128': [-83993.7883, -9, 0, '9355.332222', decimal.Decimal('5345.79745902883')],
'complex32': [complex(8.0, -32.0), 0, 10.5, 93],
'complex64': [complex(27.0, -8.0), 0, -83.5, 134],
'timestamp': [Timestamp.now(7), datetime.datetime.today()]
}
class TestConversion(unittest.TestCase):
""" Test conversions of Python values to SPL attributes/types.
"""
def setUp(self):
Tester.setup_standalone(self)
def test_bad_from_string(self):
for dt in SPL_TYPES:
topo = Topology()
schema = StreamSchema('tuple<' + dt + ' a>')
s = topo.source(['ABC'])
c = s.map(lambda x : (x,), schema=schema)