async def check_for_error(resp: aiohttp.ClientResponse,
ignore: List[ERRORS] = None):
if resp.status == 200:
return # OK
try:
error = await resp.json()
if ignore is not None:
for error_type in ignore:
if error_type.value in error["message"]:
return # OK
except aiohttp.ContentTypeError:
raise errors.DataError("[%d] invalid json response \"%s\"" %
(resp.status, await resp.text()))
raise errors.DataError("[%d]: %s" % (resp.status, error["message"]))
async def _path_info(self,
streams: List[DataStream]) -> Dict[str, StreamInfo]:
""" set path to None to list all streams """
url = "{server}/stream/list".format(server=self.server)
paths = []
if len(streams) > 6:
paths = "/joule/%"
else:
for stream in streams:
paths += [f'/joule/{stream.id}', f'/joule/{stream.id}~decim%']
params = {"extended": 1, "path": paths}
async with self._get_client() as session:
async with session.get(url, params=params) as resp:
body = await resp.text()
if resp.status != 200: # pragma: no cover
raise errors.DataError(body)
data = json.loads(body)
info = {}
for item in data:
info[item[0]] = StreamInfo(start=item[2],
end=item[3],
rows=item[4],
bytes=item[4] *
bytes_per_row(item[1]),
total_time=item[5])
return info
async def consolidate(self, stream: 'DataStream', start: int, end: int,
max_gap: int) -> int:
# remove interval gaps less than or equal to max_gap duration (in us)
intervals = await self.intervals(stream, start, end)
if len(intervals) == 0:
return # no data, nothing to do
duration = [intervals[0][0], intervals[-1][1]]
gaps = interval_tools.interval_difference([duration], intervals)
if len(gaps) == 0:
return # no interval breaks, nothing to do
small_gaps = [gap for gap in gaps if (gap[1] - gap[0]) <= max_gap]
# spawn an inserter to close each gap
info = await self._path_info([stream])
all_paths = info.keys()
path = compute_path(stream)
insert_url = "{server}/stream/insert".format(server=self.server)
for gap in small_gaps:
async with self._get_client() as session:
params = {
"start": "%d" % gap[0],
"end": "%d" % gap[1],
"path": path,
"binary": '1'
}
async with session.put(insert_url, params=params,
data=None) as resp:
if resp.status != 200: # pragma: no cover
error = await resp.text()
raise errors.DataError("NilmDB(d) error: %s" % error)
return len(small_gaps)
async def extract(self,
stream: DataStream,
start: Optional[int],
end: Optional[int],
callback: Callable[[np.ndarray, str, int], Coroutine],
max_rows: int = None,
decimation_level=None):
# figure out appropriate decimation level
if decimation_level is None:
if max_rows is None:
decimation_level = 1
else:
# find out how much data this represents
count = await self._count_by_path(compute_path(stream), start,
end)
if count > 0:
desired_decimation = np.ceil(count / max_rows)
decimation_level = 4**np.ceil(
np.log(desired_decimation) /
np.log(self.decimation_factor))
else:
# create an empty array with the right data type
data = np.array([],
dtype=pipes.compute_dtype(stream.layout))
await callback(data, stream.layout, 1)
return
# make sure the target decimation level exists and has data
try:
path = compute_path(stream, decimation_level)
if (await self._count_by_path(path, start, end)) == 0:
# no data in the decimated path
raise errors.InsufficientDecimationError(
"required level is empty")
except errors.DataError as e:
if ERRORS.NO_SUCH_STREAM.value in str(e):
# no decimated data or required level does not exist
raise errors.InsufficientDecimationError(
"required level %d does not exist" %
decimation_level)
# some other error, propogate it up
raise e # pragma: no cover
elif max_rows is not None:
# two constraints, make sure we aren't going to return too much data
count = await self._count_by_path(
compute_path(stream, decimation_level), start, end)
if count > max_rows:
raise errors.InsufficientDecimationError(
"actual_rows(%d) > max_rows(%d)" % (count, max_rows))
# retrieve data from stream
path = compute_path(stream, decimation_level)
if decimation_level > 1:
layout = stream.decimated_layout
else:
layout = stream.layout
try:
await self._extract_by_path(path, start, end, layout, callback)
except aiohttp.ClientError as e:
raise errors.DataError(str(e))
async def insert(self, stream: DataStream, start: int, end: int,
data: np.array) -> None:
"""insert stream data"""
url = "{server}/stream/insert".format(server=self.server)
params = {
"start": "%d" % start,
"end": "%d" % end,
"path": compute_path(stream),
"binary": '1'
}
async with self._get_client() as session:
async with session.put(url, params=params,
data=data.tobytes()) as resp:
if resp.status != 200:
if resp.status == 400:
error = await resp.json()
# nilmdb rejected the data
raise errors.DataError(error["message"])
raise errors.DataError(await
resp.text()) # pragma: no cover
async def initialize(self, streams: List[DataStream]) -> None:
self.connector = aiohttp.TCPConnector()
url = "{server}/stream/create".format(server=self.server)
try:
async with self._get_client() as session:
for stream in streams:
data = {
"path": compute_path(stream),
"layout": stream.layout
}
async with session.post(url, data=data) as resp:
await check_for_error(
resp, ignore=[ERRORS.STREAM_ALREADY_EXISTS])
except aiohttp.ClientError:
raise errors.DataError("cannot contact NilmDB at [%s]" %
self.server)
async def process(self, data: np.ndarray) -> None:
"""decimate data and insert it, retry on error"""
while True:
try:
if not self.path_created:
await self._create_path()
self.path_created = True
async with self._get_client() as session:
decim_data = self._process(data)
if len(decim_data) == 0:
return
if self.last_ts is not None:
start = self.last_ts
else:
start = decim_data['timestamp'][0]
end = decim_data['timestamp'][-1] + 1
self.last_ts = end
# lazy initialization of child
if self.child is None:
self.child = NilmdbDecimator(self.server, self.stream, self.level,
self.factor, self._get_client)
params = {"start": "%d" % start,
"end": "%d" % end,
"path": self.path,
"binary": '1'}
async with session.put(self.insert_url, params=params,
data=decim_data.tobytes()) as resp:
if resp.status != 200: # pragma: no cover
error = await resp.text()
raise errors.DataError("NilmDB(d) error: %s" % error)
# feed data to child decimator
await self.child.process(decim_data)
await asyncio.sleep(self.holdoff)
break # success, leave the loop
except aiohttp.ClientError as e: # pragma: no cover
log.warning("NilmDB decimation error: %r, retrying request" % e)
await asyncio.sleep(self.retry_interval) # retry the request
except asyncio.CancelledError: # pragma: no cover
break
async def run(self, pipe: pipes.Pipe) -> None:
"""insert stream data from the queue until the queue is empty"""
# create the database path
# lazy stream creation,
try:
await self._create_path()
except asyncio.CancelledError:
return
cleaner_task: Optional[asyncio.Task] = None
if self.stream.keep_us != DataStream.KEEP_ALL:
cleaner_task = asyncio.create_task(self._clean())
cleaner_task.set_name("NilmDB Clean Task for [%s]" % self.path)
while True:
try:
async with self._get_client() as session:
last_ts = None
while True:
await asyncio.sleep(self.insert_period)
data = await pipe.read()
# there might be an interval break and no new data
if len(data) > 0:
if last_ts is not None:
start = last_ts
else:
start = data['timestamp'][0]
end = data['timestamp'][-1] + 1
last_ts = end
# lazy initialization of decimator
if self.stream.decimate and self.decimator is None:
self.decimator = NilmdbDecimator(self.server, self.stream, 1, 4,
self._get_client)
# send the data
params = {"start": "%d" % start,
"end": "%d" % end,
"path": self.path,
"binary": '1'}
async with session.put(self.insert_url, params=params,
data=data.tobytes()) as resp:
if resp.status != 200:
error = await resp.text()
if cleaner_task is not None:
cleaner_task.cancel()
await cleaner_task
raise errors.DataError("NilmDB error: %s" % error)
# this was successful so consume the data
pipe.consume(len(data))
# decimate the data
if self.decimator is not None:
await self.decimator.process(data)
# check for interval breaks
if pipe.end_of_interval:
last_ts = None
if self.decimator is not None:
self.decimator.close_interval()
except aiohttp.ClientError as e: # pragma: no cover
log.warning("NilmDB raw inserter error: %r, retrying request" % e)
await asyncio.sleep(self.retry_interval) # retry the request
except (pipes.EmptyPipe, asyncio.CancelledError) as e:
break # terminate the inserter
if cleaner_task is not None:
cleaner_task.cancel()
await cleaner_task