def _get_measurements_block(self, *,
device_data: lnetatmo.WeatherStationData,
device_id: str,
module_id: str,
measurements: ty.Sequence[str],
utc_period: UtcPeriod = None
) -> TsVector:
"""Get data for a specific device and set of measurements. utc_period is the timespan for which we ask for
data, but it is optional, as utc_period=None asks for the longest possible timespan of data.
NB: Calls are limited to 1024 values. Must split to get all data in period (50 req pr sec, 500 req pr hour).
Args:
device_id: Unique identifier for the netatmo device.
module_id: Unique identifier for the netatmo module (can be None, '').
measurements: A ty.Sequence of strings representing the measurements we want to fetch.
utc_period: Inclusive start/end. The period we want data for (if none, the longest possible period
(up to 1024 values).
Returns:
A TsVector with timeseries containing data for each measurement type, in the order of the input.
"""
date_start = float(utc_period.start) if utc_period else None
date_end = float(utc_period.end) if utc_period else None
self.wait_for_rate_limiters()
self.add_action_timestamp_to_rate_limiters(utctime_now())
measurement_types_str = ','.join([m for m in measurements])
data = device_data.getMeasure(
device_id=device_id,
module_id=module_id,
scale='max',
mtype=measurement_types_str,
date_begin=date_start,
date_end=date_end)
if not data['body']:
# noinspection PyArgumentList
output = [TimeSeries() for _ in measurements]
else:
t = [float(timestamp) for timestamp in data['body'].keys()]
# Add an additional timestep fmod(dt) forward in time to indicate the validness of the last value.
dt_list = [t2 - t1 for t1, t2 in zip(t[0:-2], t[1:-1])]
dt_mode = max(set(dt_list), key=dt_list.count)
ta = TimeAxisByPoints(t + [t[-1] + dt_mode])
values_pr_time = [value for value in data['body'].values()]
values = list(map(list, zip(*values_pr_time)))
# Remove nan:
output = [TimeSeries(ta, self.set_none_to_nan(vector), POINT_INSTANT_VALUE) for vector in values]
return TsVector(output)
def test_dts_client(dtss):
dtss.start()
timeseries = ['mock1://something/1', 'mock2://something_else/2', 'mock1://something_strange/3']
try:
c = DtsClient(dtss.address)
tsv_in = TsVector([TimeSeries(ts_id) for ts_id in timeseries])
period = UtcPeriod(time(0), time(10))
tsv = c.evaluate(tsv_in, period)
assert tsv
finally:
dtss.stop()
def read_callback(self, ts_ids: StringVector,
read_period: UtcPeriod) -> TsVector:
"""This callback is passed as the default read_callback for a shyft.time_series.DtsServer.
Args:
ts_ids: A sequence of strings identifying specific timeseries available from the netatmo login. Matches the
formatting provided by DataCollectionRepository.create_ts_id()
read_period: A period defined by a utc timestamp for the start and end of the analysis period.
Returns:
A TsVector containing the resulting timeseries containing data enough to cover the query period.
"""
tsv = TsVector()
for value, ts_id in enumerate(ts_ids):
ts_info = self.parse_ts_id(ts_id=ts_id)
tsv.append(
create_ts(read_period=read_period,
value=float(ts_info['value'])))
return tsv
def read_callback(self, ts_ids: StringVector,
read_period: UtcPeriod) -> TsVector:
"""This callback is passed as the default read_callback for a shyft.time_series.DtsServer.
Args:
ts_ids: A sequence of strings identifying specific timeseries available from the netatmo login. Matches the
formatting provided by DataCollectionRepository.create_ts_id()
read_period: A period defined by a utc timestamp for the start and end of the analysis period.
Returns:
A TsVector containing the resulting timeseries containing data enough to cover the query period.
"""
logging.info(
f'DtssHost Heartbeat read_callback at {self.host.address}.')
# noinspection PyArgumentList
tsv = TsVector()
for _ in ts_ids:
tsv.append(create_ts(read_period=read_period, value=1))
return tsv
def get_measurements(self, *,
device_data: lnetatmo.WeatherStationData,
station_id: str,
module_id: str,
measurements: ty.Sequence[str],
utc_period: UtcPeriod
) -> TsVector:
"""Get data for a specific device and set of measurements. utc_period defines the data period.
Netatmo only returns data in 1024 point chunks, so this method performs multiple calls to retrieve all queried
data.
Args:
station_id: Unique identifier for the netatmo device.
module_id: Unique identifier for the netatmo module (can be None, '').
measurements: A ty.Sequence of strings representing the measurements we want to fetch.
utc_period: Inclusive start/end. The period we want data for (if none, the longest possible period
(up to 1024 values).
Returns:
A TsVector with timeseries containing data for each measurement type, in the order of the input.
"""
result_end = utc_period.start
# noinspection PyArgumentList
output = [TimeSeries() for _ in measurements]
while result_end < utc_period.end:
utc_period = UtcPeriod(result_end, utc_period.end) # Define a UtcPeriod for the remaining data.
result = self._get_measurements_block(
device_data=device_data,
device_id=station_id,
module_id=module_id,
measurements=measurements,
utc_period=utc_period)
if not any([bool(ts) for ts in result]): # None data in period. Return blank.
break
for ind, res in enumerate(result):
if res:
if not output[ind]:
output[ind] = res
else:
output[ind] = output[ind].extend(res)
result_end = result[0].time_axis.time_points_double[-1] # Set the start of the new calls UtcPeriod.
# noinspection PyArgumentList
logging.info(f'Got {len(result[0])} data points from '
f'{self.utc.to_string(result[0].time_axis.time_points_double[0])} to '
f'{self.utc.to_string(result_end)}')
return TsVector(output)
def test_dts_client_heartbeat(dtss):
dtss.start()
try:
c = DtsClient(dtss.address)
heartbeat = create_heartbeat_request('test')
tsv_in = TsVector([TimeSeries(heartbeat)])
period = UtcPeriod(time(0), time(10))
tsv = c.evaluate(tsv_in, period)
assert tsv
tsiv = c.find(heartbeat)
assert tsiv[0].name == 'heartbeat: test'
finally:
dtss.stop()
def read_callback(self, ts_ids: StringVector, read_period: UtcPeriod) -> TsVector:
"""This callback is passed as the default read_callback for a shyft.time_series.DtsServer.
Args:
ts_ids: A sequence of strings identifying specific timeseries available from the netatmo login.
read_period: A period defined by a utc timestamp for the start and end of the analysis period.
Returns:
A TsVector containing the resulting timeseries containing data enough to cover the query period.
"""
device_data, domain = self.create_netatmo_connection()
data = dict() # Group ts_ids by repo.name (scheme).
for enum, ts_id in enumerate(ts_ids):
ts_id_props = parse_ts_id(ts_id=ts_id)
measurement = domain.get_measurement(**ts_id_props)
if measurement.module.id not in data:
data[measurement.module.id] = []
data[measurement.module.id].append(
dict(enum=enum, ts=None, measurement=measurement))
for module_id in data:
measurements = [meas['measurement'] for meas in data[module_id]]
if module_id == measurements[0].station.id: # The current module is the actual station itself.
module_id_arg = None
else:
module_id_arg = module_id
measurement_types = [m.data_type.name for m in measurements]
tsvec = self.get_measurements(device_data=device_data,
station_id=measurements[0].station.id,
module_id=module_id_arg,
measurements=measurement_types,
utc_period=read_period
)
for index, ts in enumerate(tsvec):
data[module_id][index]['ts'] = ts
# Collapse nested lists and sort by initial enumerate:
transpose_data = []
for items in data.values():
transpose_data.extend(items)
sort = sorted(transpose_data, key=lambda item: item['enum'])
return TsVector([item['ts'] for item in sort])
def read_callback(self, ts_ids: StringVector,
read_period: UtcPeriod) -> TsVector:
"""DtssHost.read_callback accepts a set of urls identifying timeseries and a read period and returns bound
TimeSeries in a TsVector that contain data at least covering the read_period.
Args:
ts_ids: A sequence of strings identifying specific timeseries available from the underlying
DataCollectionRepository's.
read_period: A period defined by a utc timestamp for the start and end of the analysis period.
Returns:
A TsVector containing the resulting timeseries containing data enough to cover the query period.
"""
logging.info(
f'DtssHost received read_callback for {len(ts_ids)} ts_ids for period {read_period}.'
)
data = dict() # Group ts_ids by repo.name (scheme).
for enum, ts_id in enumerate(ts_ids):
repo_name = self.get_repo_name_from_url(ts_id)
if repo_name not in data:
data[repo_name] = []
data[repo_name].append(dict(enum=enum, ts_id=ts_id, ts=None))
for repo_name in data:
tsvec = self.repos[repo_name].read_callback(
ts_ids=StringVector([ts['ts_id'] for ts in data[repo_name]]),
read_period=read_period)
for index, ts in enumerate(tsvec):
data[repo_name][index]['ts'] = ts
# Collapse nested lists and sort by initial enumerate:
transpose_data = []
for items in data.values():
transpose_data.extend(items)
sort = sorted(transpose_data, key=lambda item: item['enum'])
return TsVector([item['ts'] for item in sort])
{'data': domain.get_measurement(station_name=station, data_type=types.temperature.name, module_name=module),
'color': '#E64C3E'}, # red
{'data': domain.get_measurement(station_name=station, data_type=types.co2.name, module_name=module),
'color': '#B0CA55'}, # green
{'data': domain.get_measurement(station_name=station, data_type=types.humidity.name, module_name=module),
'color': '#0F2933'}, # dark green
]
# ('Pressure', 'mbar', point_fx.POINT_INSTANT_VALUE, '#33120F'), # brown
# ('Noise', 'db', point_fx.POINT_INSTANT_VALUE, '#E39C30'), # yellow
# ('Rain', 'mm', point_fx.POINT_INSTANT_VALUE, '#448098'), # light blue
# ('WindStrength', 'km / h', point_fx.POINT_INSTANT_VALUE, '#8816AB'), # purple
# Get timeseries from measurements:
client = DtsClient(f'{os.environ["DTSS_SERVER"]}:{os.environ["DTSS_PORT_NUM"]}')
# client = DtsClient(f'{socket.gethostname()}:{os.environ["DTSS_PORT_NUM"]}')
tsv = TsVector([meas['data'].time_series for meas in plot_data])
cal = Calendar('Europe/Oslo')
epsilon = 0.1
now = utctime_now()
period = UtcPeriod(now - cal.DAY*3, now)
data = client.evaluate(tsv, period)
try:
fig = figure(title=f'Demo plot {cal.to_string(now)}', height=400, width=1400, x_axis_type='datetime')
fig.line([1, 2, 3, 4, 5], [5, 3, 4, 2, 1])
fig.yaxis.visible = False
fig.xaxis.formatter = DatetimeTickFormatter(
months=["%Y %b"],
days=["%F %H:%M"],