def test__stream_good_dict_of_index_and_series_data_transformed(self):
df = pd.DataFrame(
index=pd.date_range('now', periods=30, freq='T'),
columns=['A'],
data=np.cumsum(np.random.standard_normal(30), axis=0)
)
ds = ColumnDataSource(data={'index': convert_datetime_array(df.index.values),
'A': df.A})
ds._document = "doc"
stuff = {}
mock_setter = object()
def mock(*args, **kw):
stuff['args'] = args
stuff['kw'] = kw
ds.data._stream = mock
new_df = pd.DataFrame(
index=df.index + pd.to_timedelta('30m'),
columns=df.columns,
data=np.random.standard_normal(30)
)
ds._stream({'index': new_df.index, 'A': new_df.A}, "foo", mock_setter)
self.assertTrue(np.array_equal(stuff['args'][2]['index'],
convert_datetime_array(new_df.index.values)))
self.assertTrue(np.array_equal(stuff['args'][2]['A'], new_df.A.values))
def test__stream_good_dict_of_index_and_series_data_transformed(
self, pd) -> None:
df = pd.DataFrame(index=pd.date_range('now', periods=30, freq='T'),
columns=['A'],
data=np.cumsum(np.random.standard_normal(30),
axis=0))
ds = bms.ColumnDataSource(
data={
'index': convert_datetime_array(df.index.values),
'A': df.A
})
ds._document = "doc"
stuff = {}
mock_setter = object()
def mock(*args, **kw):
stuff['args'] = args
stuff['kw'] = kw
ds.data._stream = mock
new_df = pd.DataFrame(index=df.index + pd.to_timedelta('30m'),
columns=df.columns,
data=np.random.standard_normal(30))
ds._stream({'index': new_df.index, 'A': new_df.A}, "foo", mock_setter)
assert np.array_equal(stuff['args'][2]['index'],
convert_datetime_array(new_df.index.values))
assert np.array_equal(stuff['args'][2]['A'], new_df.A.values)
def _update_datasource(self, source, data):
"""
Update datasource with data for a new frame.
"""
data = {k: decode_bytes(vs) for k, vs in data.items()}
empty = all(len(v) == 0 for v in data.values())
if (self.streaming
and self.streaming[0].data is self.current_frame.data
and self._stream_data and not empty):
stream = self.streaming[0]
if stream._triggering:
data = {k: v[-stream._chunk_length:] for k, v in data.items()}
# NOTE: Workaround for bug in bokeh 0.12.14, data conversion
# should be removed once fixed in bokeh (https://github.com/bokeh/bokeh/issues/7587)
converted_data = {}
for k, vals in data.items():
cdata = source.data[k]
odata = data[k]
if (bokeh_version in ['0.12.14', '0.12.15dev1']
and isinstance(cdata, np.ndarray)
and cdata.dtype.kind == 'M'
and isinstance(vals, np.ndarray)
and vals.dtype.kind == 'M'):
cdata = convert_datetime_array(cdata)
odata = convert_datetime_array(odata)
if len(odata):
cdata = np.concatenate([odata, cdata])
converted_data[k] = cdata
if converted_data:
for k, vals in data.items():
cdata = source.data[k]
odata = data[k]
if k not in converted_data:
if len(odata):
cdata = np.concatenate([odata, cdata])
converted_data[k] = cdata
source.data.update(converted_data)
else:
source.stream(data, stream.length)
return
# Determine if the CDS.data requires a full replacement or simply needs
# to be updated. A replacement is required if untouched columns
# are not the same length as the columns being updated.
current_length = [
len(v) for v in source.data.values()
if isinstance(v, (list, np.ndarray))
]
new_length = [
len(v) for v in data.values() if isinstance(v, (list, np.ndarray))
]
untouched = [k for k in source.data if k not in data]
if (untouched and current_length and new_length
and current_length[0] != new_length[0]):
source.data = data
else:
source.data.update(data)
def test_convert_datetime_type_array() -> None:
a = np.array(['2018-01-03T15:37:59', '2018-01-03T15:37:59.922452', '2016-05-11'], dtype='datetime64')
r = bus.convert_datetime_array(a)
assert r[0] == 1514993879000.0
assert r[1] == 1514993879922.452
assert r[2] == 1462924800000.0
assert r.dtype == 'float64'
def test_convert_datetime_type_array():
a = np.array(['2018-01-03T15:37:59', '2018-01-03T15:37:59.922452', '2016-05-11'], dtype='datetime64')
r = bus.convert_datetime_array(a)
assert r[0] == 1514993879000.0
assert r[1] == 1514993879922.452
assert r[2] == 1462924800000.0
assert r.dtype == 'float64'
def test__stream_good_datetime64_data_transformed(self):
now = dt.datetime.now()
dates = np.array([now+dt.timedelta(i) for i in range(1, 10)], dtype='datetime64')
dates = convert_datetime_array(dates)
ds = ColumnDataSource(data=dict(index=dates, b=list(range(1, 10))))
ds._document = "doc"
stuff = {}
mock_setter = object()
def mock(*args, **kw):
stuff['args'] = args
stuff['kw'] = kw
ds.data._stream = mock
# internal implementation of stream
new_date = np.array([now+dt.timedelta(10)], dtype='datetime64')
ds._stream(dict(index=new_date, b=[10]), "foo", mock_setter)
transformed_date = convert_datetime_array(new_date)
self.assertTrue(np.array_equal(stuff['args'][2]['index'], transformed_date))
def _update_datasource(self, source, data):
"""
Update datasource with data for a new frame.
"""
data = {k: decode_bytes(vs) for k, vs in data.items()}
if (self.streaming
and self.streaming[0].data is self.current_frame.data
and self._stream_data):
stream = self.streaming[0]
if stream._triggering:
data = {k: v[-stream._chunk_length:] for k, v in data.items()}
# NOTE: Workaround for bug in bokeh 0.12.14, data conversion
# should be removed once fixed in bokeh (https://github.com/bokeh/bokeh/issues/7587)
converted_data = {}
for k, vals in data.items():
cdata = source.data[k]
odata = data[k]
if (bokeh_version in ['0.12.14', '0.12.15dev1']
and isinstance(cdata, np.ndarray)
and cdata.dtype.kind == 'M'
and isinstance(vals, np.ndarray)
and vals.dtype.kind == 'M'):
cdata = convert_datetime_array(cdata)
odata = convert_datetime_array(odata)
if len(odata):
cdata = np.concatenate([odata, cdata])
converted_data[k] = cdata
if converted_data:
for k, vals in data.items():
cdata = source.data[k]
odata = data[k]
if k not in converted_data:
if len(odata):
cdata = np.concatenate([odata, cdata])
converted_data[k] = cdata
source.data.update(converted_data)
else:
source.stream(data, stream.length)
else:
source.data.update(data)
def test_transform_series_force_list_default_with_buffers(pd) -> None:
# default int seems to be int64, can't be converted to buffer!
df = pd.Series([1, 3, 5, 6, 8])
out = bus.transform_series(df)
assert isinstance(out, list)
assert out == [1, 3, 5, 6, 8]
df = pd.Series([1, 3, 5, 6, 8], dtype=np.int32)
bufs = []
out = bus.transform_series(df, buffers=bufs)
assert isinstance(out, dict)
assert len(bufs) == 1
assert len(bufs[0]) == 2
assert isinstance(bufs[0][0], dict)
assert list(bufs[0][0]) == ["id"]
assert bufs[0][1] == np.array(df).tobytes()
assert 'shape' in out
assert out['shape'] == df.shape
assert 'dtype' in out
assert out['dtype'] == df.dtype.name
assert '__buffer__' in out
df = pd.Series([1.0, 3, 5, 6, 8])
bufs = []
out = bus.transform_series(df, buffers=bufs)
assert isinstance(out, dict)
assert len(bufs) == 1
assert len(bufs[0]) == 2
assert isinstance(bufs[0][0], dict)
assert list(bufs[0][0]) == ["id"]
assert bufs[0][1] == np.array(df).tobytes()
assert 'shape' in out
assert out['shape'] == df.shape
assert 'dtype' in out
assert out['dtype'] == df.dtype.name
assert '__buffer__' in out
df = pd.Series(np.array([np.nan, np.inf, -np.inf, 0]))
bufs = []
out = bus.transform_series(df, buffers=bufs)
assert isinstance(out, dict)
assert len(bufs) == 1
assert len(bufs[0]) == 2
assert isinstance(bufs[0][0], dict)
assert list(bufs[0][0]) == ["id"]
assert bufs[0][1] == np.array(df).tobytes()
assert 'shape' in out
assert out['shape'] == df.shape
assert 'dtype' in out
assert out['dtype'] == df.dtype.name
assert '__buffer__' in out
# PeriodIndex
df = pd.period_range('1900-01-01','2000-01-01', freq='A')
bufs = []
out = bus.transform_series(df, buffers=bufs)
assert isinstance(out, dict)
assert len(bufs) == 1
assert len(bufs[0]) == 2
assert isinstance(bufs[0][0], dict)
assert list(bufs[0][0]) == ["id"]
assert bufs[0][1] == bus.convert_datetime_array(df.to_timestamp().values).tobytes()
assert 'shape' in out
assert out['shape'] == df.shape
assert 'dtype' in out
assert out['dtype'] == 'float64'
assert '__buffer__' in out
# DatetimeIndex
df = pd.period_range('1900-01-01','2000-01-01', freq='A').to_timestamp()
bufs = []
out = bus.transform_series(df, buffers=bufs)
assert isinstance(out, dict)
assert len(bufs) == 1
assert len(bufs[0]) == 2
assert isinstance(bufs[0][0], dict)
assert list(bufs[0][0]) == ["id"]
assert bufs[0][1] == bus.convert_datetime_array(df.values).tobytes()
assert 'shape' in out
assert out['shape'] == df.shape
assert 'dtype' in out
assert out['dtype'] == 'float64'
assert '__buffer__' in out
# TimeDeltaIndex
df = pd.to_timedelta(np.arange(5), unit='s')
bufs = []
out = bus.transform_series(df, buffers=bufs)
assert isinstance(out, dict)
assert len(bufs) == 1
assert len(bufs[0]) == 2
assert isinstance(bufs[0][0], dict)
assert list(bufs[0][0]) == ["id"]
assert bufs[0][1] == bus.convert_datetime_array(df.values).tobytes()
assert 'shape' in out
assert out['shape'] == df.shape
assert 'dtype' in out
assert out['dtype'] == 'float64'
assert '__buffer__' in out
def test_convert_datetime_type_array_ignores_non_datetime_array() -> None:
a = np.arange(0,10,100)
assert bus.convert_datetime_array(a) is a
def test_convert_datetime_type_array_ignores_non_array(obj) -> None:
assert bus.convert_datetime_array(obj) is obj
def test_transform_series_force_list_default_with_buffers(pd):
# default int seems to be int64, can't be converted to buffer!
df = pd.Series([1, 3, 5, 6, 8])
out = bus.transform_series(df)
assert isinstance(out, list)
assert out == [1, 3, 5, 6, 8]
df = pd.Series([1, 3, 5, 6, 8], dtype=np.int32)
bufs = []
out = bus.transform_series(df, buffers=bufs)
assert isinstance(out, dict)
assert len(bufs) == 1
assert len(bufs[0]) == 2
assert isinstance(bufs[0][0], dict)
assert list(bufs[0][0]) == ["id"]
assert bufs[0][1] == np.array(df).tobytes()
assert 'shape' in out
assert out['shape'] == df.shape
assert 'dtype' in out
assert out['dtype'] == df.dtype.name
assert '__buffer__' in out
df = pd.Series([1.0, 3, 5, 6, 8])
bufs = []
out = bus.transform_series(df, buffers=bufs)
assert isinstance(out, dict)
assert len(bufs) == 1
assert len(bufs[0]) == 2
assert isinstance(bufs[0][0], dict)
assert list(bufs[0][0]) == ["id"]
assert bufs[0][1] == np.array(df).tobytes()
assert 'shape' in out
assert out['shape'] == df.shape
assert 'dtype' in out
assert out['dtype'] == df.dtype.name
assert '__buffer__' in out
df = pd.Series(np.array([np.nan, np.inf, -np.inf, 0]))
bufs = []
out = bus.transform_series(df, buffers=bufs)
assert isinstance(out, dict)
assert len(bufs) == 1
assert len(bufs[0]) == 2
assert isinstance(bufs[0][0], dict)
assert list(bufs[0][0]) == ["id"]
assert bufs[0][1] == np.array(df).tobytes()
assert 'shape' in out
assert out['shape'] == df.shape
assert 'dtype' in out
assert out['dtype'] == df.dtype.name
assert '__buffer__' in out
# PeriodIndex
df = pd.period_range('1900-01-01','2000-01-01', freq='A')
bufs = []
out = bus.transform_series(df, buffers=bufs)
assert isinstance(out, dict)
assert len(bufs) == 1
assert len(bufs[0]) == 2
assert isinstance(bufs[0][0], dict)
assert list(bufs[0][0]) == ["id"]
assert bufs[0][1] == bus.convert_datetime_array(df.to_timestamp().values).tobytes()
assert 'shape' in out
assert out['shape'] == df.shape
assert 'dtype' in out
assert out['dtype'] == 'float64'
assert '__buffer__' in out
# DatetimeIndex
df = pd.period_range('1900-01-01','2000-01-01', freq='A').to_timestamp()
bufs = []
out = bus.transform_series(df, buffers=bufs)
assert isinstance(out, dict)
assert len(bufs) == 1
assert len(bufs[0]) == 2
assert isinstance(bufs[0][0], dict)
assert list(bufs[0][0]) == ["id"]
assert bufs[0][1] == bus.convert_datetime_array(df.values).tobytes()
assert 'shape' in out
assert out['shape'] == df.shape
assert 'dtype' in out
assert out['dtype'] == 'float64'
assert '__buffer__' in out
# TimeDeltaIndex
df = pd.to_timedelta(np.arange(5), unit='s')
bufs = []
out = bus.transform_series(df, buffers=bufs)
assert isinstance(out, dict)
assert len(bufs) == 1
assert len(bufs[0]) == 2
assert isinstance(bufs[0][0], dict)
assert list(bufs[0][0]) == ["id"]
assert bufs[0][1] == bus.convert_datetime_array(df.values).tobytes()
assert 'shape' in out
assert out['shape'] == df.shape
assert 'dtype' in out
assert out['dtype'] == 'float64'
assert '__buffer__' in out
def test_convert_datetime_type_array_ignores_non_datetime_array():
a = np.arange(0,10,100)
assert bus.convert_datetime_array(a) is a
def test_convert_datetime_type_array_ignores_non_array(obj):
assert bus.convert_datetime_array(obj) is obj
