def test_data_collection_period_relative():
collection = DataCollectionPeriodRelative(start_offset=3600,
wait_time=10,
end_offset=3600 / 2)
assert collection.period().start - st.utctime_now() - 3600 < 0.01
assert collection.period().end - st.utctime_now() - 3600 / 2 < 0.01
def next(self) -> ty.Dict[str, Number]:
self.client = st.DtsClient(
f'{os.environ["DTSS_SERVER"]}:{os.environ["DTSS_PORT_NUM"]}')
now = st.utctime_now()
period = st.UtcPeriod(now - self.cal.HOUR, now)
data = self.client.evaluate(self.tsv, period)
if self.type == 'temp':
ts = data[0]
elif self.type == 'co2':
ts = data[1]
else:
ts = data[2]
values = ts.values.to_numpy()
times = ts.time_axis.time_points_double
out = dict(
current=values[-1] if len(values) else np.nan,
time=float(st.utctime_now()) * 1000,
# time=float(times[0]) * 1000,
max=max(values),
min=min(values),
)
# print('data fetch success')
return out
def check_next_rate(self) -> bool:
"""Check if a hypothetical next action (performed right now) would exceed the rate limit."""
now = utctime_now()
next_timestamps = list(self.action_timestamps)[1:]
next_timestamps.append(now)
return self._check_rate(action_timestamps=next_timestamps,
action_limit=self.action_limit,
timespan=self.timespan)
def update():
# Update dashboard data.
period = st.UtcPeriod(st.utctime_now() - st.Calendar.DAY,
st.utctime_now())
data = client.evaluate(
st.TsVector([
domain.get_measurement(station_name='Eftasåsen',
module_name='Ute',
data_type='Temperature').time_series
]), period)
time, temp = get_xy(cal, data[0])
new = {
'temp': [temp[-1]],
'time': [bokeh_time_from_timestamp(cal, st.utctime_now())],
'color': ['blue']
}
now_source.stream(new, 100)
def _get_measurements_block(self, *,
device_data: lnetatmo.WeatherStationData,
device_id: str,
module_id: str,
measurements: ty.Sequence[str],
utc_period: UtcPeriod = None
) -> TsVector:
"""Get data for a specific device and set of measurements. utc_period is the timespan for which we ask for
data, but it is optional, as utc_period=None asks for the longest possible timespan of data.
NB: Calls are limited to 1024 values. Must split to get all data in period (50 req pr sec, 500 req pr hour).
Args:
device_id: Unique identifier for the netatmo device.
module_id: Unique identifier for the netatmo module (can be None, '').
measurements: A ty.Sequence of strings representing the measurements we want to fetch.
utc_period: Inclusive start/end. The period we want data for (if none, the longest possible period
(up to 1024 values).
Returns:
A TsVector with timeseries containing data for each measurement type, in the order of the input.
"""
date_start = float(utc_period.start) if utc_period else None
date_end = float(utc_period.end) if utc_period else None
self.wait_for_rate_limiters()
self.add_action_timestamp_to_rate_limiters(utctime_now())
measurement_types_str = ','.join([m for m in measurements])
data = device_data.getMeasure(
device_id=device_id,
module_id=module_id,
scale='max',
mtype=measurement_types_str,
date_begin=date_start,
date_end=date_end)
if not data['body']:
# noinspection PyArgumentList
output = [TimeSeries() for _ in measurements]
else:
t = [float(timestamp) for timestamp in data['body'].keys()]
# Add an additional timestep fmod(dt) forward in time to indicate the validness of the last value.
dt_list = [t2 - t1 for t1, t2 in zip(t[0:-2], t[1:-1])]
dt_mode = max(set(dt_list), key=dt_list.count)
ta = TimeAxisByPoints(t + [t[-1] + dt_mode])
values_pr_time = [value for value in data['body'].values()]
values = list(map(list, zip(*values_pr_time)))
# Remove nan:
output = [TimeSeries(ta, self.set_none_to_nan(vector), POINT_INSTANT_VALUE) for vector in values]
return TsVector(output)
def refresh_data(self, cal: st.Calendar, hist_length: st.time) -> None:
"""Put new data into datasource for icon and plot."""
period = st.UtcPeriod(st.utctime_now() - hist_length, st.utctime_now())
temp = self.dtss_data.get_data(period=period)
t, v = get_xy(cal, temp)
points = [(time, value) for time, value in zip(t, v)]
ideal_number_of_points = 10
epsilon = (len(points) / (3 * ideal_number_of_points)) * 2 # https://stackoverflow.com/questions/57052434/can-i-guess-the-appropriate-epsilon-for-rdp-ramer-douglas-peucker
reduced = rdp(points, epsilon=epsilon)
lists = list(map(list, zip(*reduced)))
t = lists[0]
v = lists[1]
new = {'value': [v],
'time': [t],
'color': ['grey']
# 'color': [self.icon.color_selector(v[-1])]
}
self.source.data = new
def find_callback(self, query: str) -> TsInfoVector:
"""This callback is passed as the default find_callback for a shyft.time_series.DtsServer.
Args:
query: The url representing a relevant query for this DataCollectionRepository. Matches the formatting
provided by DataCollectionRepository.create_ts_query()
Returns:
A sequence of results matching the query.
"""
ts = create_ts(0)
tsi = TsInfo(name=query,
point_fx=point_interpretation_policy.POINT_INSTANT_VALUE,
delta_t=0,
olson_tz_id='',
data_period=ts.time_axis.total_period(),
created=utctime_now(),
modified=utctime_now())
tsiv = TsInfoVector()
tsiv.append(tsi)
return tsiv
def dashboard(doc):
now_source = ColumnDataSource({'temp': [], 'time': [], 'color': []})
period = st.UtcPeriod(st.utctime_now() - st.Calendar.DAY, st.utctime_now())
fig = figure(
title='Show current outdoor temperature!',
x_axis_type='datetime',
sizing_mode='scale_both',
# y_range=[15, 35],
)
fig.circle(source=now_source, x='time', y='temp', color='color', size=10)
# fig.x_range = Range1d(bokeh_time_from_timestamp(cal, period.start), bokeh_time_from_timestamp(cal, period.end))
def update():
# Update dashboard data.
period = st.UtcPeriod(st.utctime_now() - st.Calendar.DAY,
st.utctime_now())
data = client.evaluate(
st.TsVector([
domain.get_measurement(station_name='Eftasåsen',
module_name='Ute',
data_type='Temperature').time_series
]), period)
time, temp = get_xy(cal, data[0])
new = {
'temp': [temp[-1]],
'time': [bokeh_time_from_timestamp(cal, st.utctime_now())],
'color': ['blue']
}
now_source.stream(new, 100)
# fig.x_range = Range1d(bokeh_time_from_timestamp(cal, period.start), bokeh_time_from_timestamp(cal, period.end))
update()
doc.add_periodic_callback(update, 1000)
doc.title = "Now with live updating!"
doc.add_root(fig)
def _check_rate(*, action_timestamps: Sequence[TimeType],
action_limit: int, timespan: TimeType) -> bool:
"""With a set of timestamps, an action limit and a timespan, check if the performed actions exceed the rate
limit or not.
Args:
action_timestamps: The history of actions represented by their timestamps.
action_limit: The maximum amount of allowed actions within timestamp.
timespan: The timespan over which action_limit is allowed.
Returns:
True when within rate, False, when rate is exceeded.
"""
if len(action_timestamps) < action_limit:
return True
now = utctime_now()
# Check if the Nth call happened less than a timespan ago:
if now - action_timestamps[-action_limit] < timespan:
return False
else:
return True
{'data': domain.get_measurement(station_name=station, data_type=types.humidity.name, module_name=module),
'color': '#0F2933'}, # dark green
]
# ('Pressure', 'mbar', point_fx.POINT_INSTANT_VALUE, '#33120F'), # brown
# ('Noise', 'db', point_fx.POINT_INSTANT_VALUE, '#E39C30'), # yellow
# ('Rain', 'mm', point_fx.POINT_INSTANT_VALUE, '#448098'), # light blue
# ('WindStrength', 'km / h', point_fx.POINT_INSTANT_VALUE, '#8816AB'), # purple
# Get timeseries from measurements:
client = DtsClient(f'{os.environ["DTSS_SERVER"]}:{os.environ["DTSS_PORT_NUM"]}')
# client = DtsClient(f'{socket.gethostname()}:{os.environ["DTSS_PORT_NUM"]}')
tsv = TsVector([meas['data'].time_series for meas in plot_data])
cal = Calendar('Europe/Oslo')
epsilon = 0.1
now = utctime_now()
period = UtcPeriod(now - cal.DAY*3, now)
data = client.evaluate(tsv, period)
try:
fig = figure(title=f'Demo plot {cal.to_string(now)}', height=400, width=1400, x_axis_type='datetime')
fig.line([1, 2, 3, 4, 5], [5, 3, 4, 2, 1])
fig.yaxis.visible = False
fig.xaxis.formatter = DatetimeTickFormatter(
months=["%Y %b"],
days=["%F %H:%M"],
hours=["%a %H:%M"],
minutes=["%H:%M"]
)
def __init__(self, start: TimeType, wait_time: TimeType, end: Optional[TimeType] = st.utctime_now()):
"""DataCollectionPeriods are used to define the period query pattern for a DataCollectionSerive. The
DataCollectionPeriod defines the start, stop of every individual period, and the wait time between each query.
Args:
start_offset: The offset in seconds from now that defines the start of the queried data period.
end_offset: The offset in seconds from now that defines the end of the queried data period. Default=0.
wait_time: The wait time in seconds between individual queries.
"""
self.wait_time = wait_time
self.start = start
self.end = end
def period(self) -> st.UtcPeriod:
"""Return a UtcPeriod correctly defined for a data query right now."""
now = st.utctime_now()
return st.UtcPeriod(now - self.start_offset, now - self.end_offset)
def perform_action(self) -> None:
"""Add now timestamp indicating an action to historical action timestamps."""
self.action_timestamps.append(utctime_now())