def create_dev_app(self) -> web.Application:
parser = argparse.ArgumentParser()
self._build_args(parser)
# must specify module config in JOULE_MODULE_CONFIG environment variable
if 'JOULE_MODULE_CONFIG' not in os.environ:
raise ConfigurationError(
"JOULE_MODULE_CONFIG not set, must specify config file")
module_config_file = os.environ['JOULE_MODULE_CONFIG']
if not os.path.isfile(module_config_file):
raise ConfigurationError(
"JOULE_MODULE_CONFIG is not a module config file")
module_args = helpers.load_args_from_file(module_config_file)
parsed_args = parser.parse_args(module_args)
self.node = api.get_node(parsed_args.node)
self.stop_requested = False
my_app = self._create_app()
async def on_startup(app):
app['task'] = await self.run_as_task(parsed_args, app)
my_app.on_startup.append(on_startup)
async def on_shutdown(app):
self.stop()
await app['task']
my_app.on_shutdown.append(on_shutdown)
return my_app
def test_builds_live_input_pipes(self):
server = FakeJoule()
src_data = create_source_data(server, is_destination=True)
self.start_server(server)
loop = asyncio.get_event_loop()
my_node = api.get_node()
async def runner():
pipes_in, pipes_out = await build_network_pipes(
{'input': '/test/source:uint8[x,y,z]'}, {}, {}, my_node, None,
None, True)
blk1 = await pipes_in['input'].read()
self.assertTrue(pipes_in['input'].end_of_interval)
pipes_in['input'].consume(len(blk1))
blk2 = await pipes_in['input'].read()
pipes_in['input'].consume(len(blk2))
rx_data = np.hstack(
(blk1, interval_token(pipes_in['input'].layout), blk2))
np.testing.assert_array_equal(rx_data, src_data)
with self.assertRaises(EmptyPipe):
await pipes_in['input'].read()
await my_node.close()
with self.assertLogs(level='INFO') as log:
loop.run_until_complete(runner())
log_dump = ' '.join(log.output)
self.stop_server()
loop.close()
def node(self):
# lazy node construction, raise error if it cannot be created
if self._node is None:
self._node = api.get_node(self.name)
click.echo("--connecting to [%s]--" % self._node.name, err=True)
self.name = self._node.name
return self._node
def test_output_errors(self):
server = FakeJoule()
create_destination(server)
create_source_data(server, is_destination=True)
self.start_server(server)
loop = asyncio.get_event_loop()
my_node = api.get_node()
# errors on layout differences
with self.assertRaises(ConfigurationError) as e:
loop.run_until_complete(
build_network_pipes({},
{'output': '/test/dest:float32[x,y,z]'},
{}, my_node, None, None))
self.assertIn('uint8_3', str(e.exception))
# errors if the stream is already being produced
with self.assertRaises(ApiError) as e:
loop.run_until_complete(
build_network_pipes({},
{'output': '/test/source:uint8[e0,e1,e2]'},
{}, my_node, None, None))
loop.run_until_complete(my_node.close())
self.assertIn('already being produced', str(e.exception))
self.stop_server()
loop.close()
def test_creates_output_stream_if_necessary(self):
server = FakeJoule()
self.start_server(server)
loop = asyncio.get_event_loop()
my_node = api.get_node()
self.assertEqual(len(server.streams), 0)
async def runner():
# the destination does not exist
with self.assertRaises(errors.ApiError):
await my_node.data_stream_get("/test/dest")
pipes_in, pipes_out = await build_network_pipes(
{}, {'output': '/test/dest:uint8[e0,e1,e2]'}, {}, my_node,
None, None)
await pipes_out['output'].close()
# make sure the stream exists
dest_stream = await my_node.data_stream_get("/test/dest")
self.assertIsNotNone(dest_stream)
await my_node.close()
with self.assertLogs(level='INFO') as log:
loop.run_until_complete(runner())
log_dump = ' '.join(log.output)
self.assertIn('creating', log_dump)
self.stop_server()
loop.close()
def test_input_errors(self):
server = FakeJoule()
create_source_data(server, is_destination=False)
self.start_server(server)
loop = asyncio.get_event_loop()
my_node = api.get_node()
# errors on layout differences
with self.assertRaises(ConfigurationError) as e:
with self.assertLogs(level='INFO'):
loop.run_until_complete(
build_network_pipes({'input': '/test/source:uint8[x,y]'},
{}, {},
my_node,
10,
20,
force=True))
self.assertIn('uint8_3', str(e.exception))
# errors if live stream is requested but unavailable
with self.assertRaises(ApiError) as e:
loop.run_until_complete(
build_network_pipes({'input': '/test/source:uint8[x,y,z]'}, {},
{}, my_node, None, None))
self.assertIn('not being produced', str(e.exception))
loop.run_until_complete(my_node.close())
self.stop_server()
loop.close()
async def setUp(self):
self.node1 = api.get_node("node1.joule")
followers = await self.node1.follower_list()
self.assertEqual(len(followers), 1)
self.node2 = followers[0]
# add the root user as a follower to node2.joule
user = await self.node2.master_add("user", "cli")
subprocess.run(
f"joule node add node2.joule https://node2.joule:8088 {user.key}".
split(" "))
"""
def test_configuration_errors(self):
server = FakeJoule()
loop = asyncio.get_event_loop()
self.start_server(server)
my_node = api.get_node()
# must specify an inline configuration
with self.assertRaises(ConfigurationError):
loop.run_until_complete(
build_network_pipes({'input': '/test/source'}, {}, {},
my_node,
10,
20,
force=True))
loop.run_until_complete(my_node.close())
self.stop_server()
loop.close()
async def _run(loop: asyncio.AbstractEventLoop):
node = api.get_node()
procs = start_standalone_procs1()
time.sleep(4) # let procs run
stop_standalone_procs(procs)
time.sleep(8) # close sockets
procs = await start_standalone_procs2(node)
time.sleep(4) # let procs run
stop_standalone_procs(procs)
time.sleep(4) # close sockets
await check_streams(node)
await check_modules(node)
await check_data(node)
await check_logs(node)
await node.close()
return
async def setUp(self):
self.node = api.get_node()
"""
└── test
├── f1
│ ├── f1a
│ │ ├── s1a1
│ │ └── s1a2
│ └── s1a
└── f2
└── f2a
└── s2a1a
"""
stream = api.DataStream(name="s1a1", elements=[api.Element(name="e1")])
await self.node.data_stream_create(stream, "/test/f1/f1a")
stream.name = "s1a2"
await self.node.data_stream_create(stream, "/test/f1/f1a")
stream.name = "s1a"
await self.node.data_stream_create(stream, "/test/f1")
stream.name = "s2a1a"
await self.node.data_stream_create(stream, "/test/f2/f2a")
def node_list(config):
"""Display authorized nodes."""
try:
nodes = api.get_nodes()
default_node = api.get_node()
except errors.ApiError as e:
raise click.ClickException(str(e))
result = []
default_indicator = click.style("\u25CF", fg="green")
for node in nodes:
if default_node.name == node.name:
name = default_indicator + " " + node.name
else:
name = node.name
result.append([name, node.url])
click.echo("List of authorized nodes ("+default_indicator+"=default)")
click.echo(tabulate(result,
headers=["Node", "URL"],
tablefmt="fancy_grid"))
def cli_copy(config: Config, start, end, replace, new, destination_node,
source, destination):
"""Copy events to a different stream."""
try:
if destination_node is None:
dest_node = config.node
else:
dest_node = get_node(destination_node)
except errors.ApiError:
raise click.ClickException(
f"Invalid destination node [{destination_node}]")
try:
asyncio.run(
_run(config.node, dest_node, start, end, new, replace, source,
destination))
except errors.ApiError as e:
raise click.ClickException(str(e)) from e
finally:
asyncio.run(config.close_node())
if destination_node is not None: # different destination node
asyncio.run(dest_node.close())
click.echo("OK")
def test_builds_output_pipes(self):
server = FakeJoule()
blk1 = helpers.create_data("uint8_3", start=10, step=1, length=10)
blk2 = helpers.create_data("uint8_3", start=200, step=1, length=10)
create_destination(server)
self.start_server(server)
my_node = api.get_node()
async def runner():
pipes_in, pipes_out = await build_network_pipes(
{}, {'output': '/test/dest:uint8[e0,e1,e2]'}, {},
my_node,
10,
210,
force=True)
await pipes_out['output'].write(blk1)
await pipes_out['output'].close_interval()
await pipes_out['output'].write(blk2)
await pipes_out['output'].close_interval()
await pipes_out['output'].close()
await my_node.close()
asyncio.run(runner())
# first msg should be the data removal
(stream_id, start, end) = self.msgs.get()
self.assertEqual(int(stream_id), 8)
self.assertEqual(int(start), 10)
self.assertEqual(int(end), 210)
# next message should be the output data
mock_entry: MockDbEntry = self.msgs.get()
np.testing.assert_array_equal(mock_entry.data[:len(blk1)], blk1)
np.testing.assert_array_equal(mock_entry.data[len(blk1):], blk2)
self.assertEqual(mock_entry.intervals, [[10, 19], [200, 209]])
self.stop_server()
async def wait_for_follower():
# wait until the local node is online
max_tries = 10
num_tries = 0
while num_tries < max_tries:
num_tries += 1
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
result = sock.connect_ex(('localhost', 8088))
sock.close()
if result == 0:
break
time.sleep(0.5)
if num_tries == max_tries:
print("error, local node is not online")
return -1
node1 = api.get_node("node1.joule")
while True:
followers = await node1.follower_list()
if len(followers) == 1:
break
await asyncio.sleep(0.5)
await node1.close()
return 0 # success
def test_warns_before_data_removal(self):
server = FakeJoule()
create_destination(server)
self.start_server(server)
loop = asyncio.get_event_loop()
my_node = api.get_node()
async def runner():
with mock.patch('joule.client.helpers.pipes.click') as mock_click:
mock_click.confirm = mock.Mock(return_value=False)
with self.assertRaises(SystemExit):
await build_network_pipes(
{}, {'output': '/test/dest:uint8[e0,e1,e2]'}, {},
my_node,
1472500708000000,
1476475108000000,
force=False)
msg = mock_click.confirm.call_args[0][0]
# check for start time
self.assertIn("29 Aug", msg)
# check for end time
self.assertIn("14 Oct", msg)
# if end_time not specified anything *after* start is removed
mock_click.confirm = mock.Mock(return_value=False)
with self.assertRaises(SystemExit):
await build_network_pipes(
{}, {'output': '/test/dest:uint8[e0,e1,e2]'}, {},
my_node,
1472500708000000,
None,
force=False)
msg = mock_click.confirm.call_args[0][0]
# check for start time
self.assertIn("29 Aug", msg)
# check for *after*
self.assertIn("after", msg)
# if start_time not specified anything *before* end is removed
mock_click.confirm = mock.Mock(return_value=False)
with self.assertRaises(SystemExit):
await build_network_pipes(
{}, {'output': '/test/dest:uint8[e0,e1,e2]'}, {},
my_node,
None,
1476475108000000,
force=False)
msg = mock_click.confirm.call_args[0][0]
# check for *before*
self.assertIn("before", msg)
# check for end time
self.assertIn("14 Oct", msg)
await my_node.close()
# suppress "cancelled" notifications when program exits
#with self.assertLogs(level='INFO'):
loop.run_until_complete(runner())
self.stop_server()
loop.close()
def start(self, parsed_args: argparse.Namespace = None):
"""
Execute the module. Do not override this function. Creates an event loop and
executes the :meth:`run` coroutine.
Args:
parsed_args: omit to parse the command line arguments
.. code-block:: python
class ModuleDemo(BaseModule):
# body of module...
# at a minimum the run coroutine must be implemented
if __name__ == "__main__":
my_module = ModuleDemo()
my_module.start()
"""
if parsed_args is None: # pragma: no cover
parser = argparse.ArgumentParser()
self._build_args(parser)
module_args = helpers.module_args()
parsed_args = parser.parse_args(module_args)
# asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
loop = asyncio.get_event_loop()
self.stop_requested = False
if parsed_args.api_socket != "unset":
# should be set by the joule daemon
if 'JOULE_CA_FILE' in os.environ:
cafile = os.environ['JOULE_CA_FILE']
else:
cafile = ""
self.node = node.UnixNode("local", parsed_args.api_socket, cafile)
else:
self.node = api.get_node(parsed_args.node)
self.runner: web.AppRunner = loop.run_until_complete(
self._start_interface(parsed_args))
app = None
if self.runner is not None:
app = self.runner.app
try:
task = loop.run_until_complete(self.run_as_task(parsed_args, app))
except ConfigurationError as e:
log.error("ERROR: " + str(e))
self._cleanup(loop)
return
def stop_task():
# give task no more than 2 seconds to exit
loop.call_later(self.STOP_TIMEOUT, task.cancel)
# run custom exit routine
self.stop()
loop.add_signal_handler(signal.SIGINT, stop_task)
loop.add_signal_handler(signal.SIGTERM, stop_task)
try:
loop.run_until_complete(task)
except (asyncio.CancelledError, pipes.EmptyPipe, EmptyPipeError) as e:
pass
finally:
self._cleanup(loop)
async def setUp(self):
self.node = api.get_node()
async def get_dsn(node_name) -> str:
node = api.get_node(node_name)
conn_info = await node.db_connection_info()
await node.close()
return conn_info.to_dsn()
async def _run(config_node,
start,
end,
new,
destination_node,
source,
destination,
source_url=None):
# determine if the source node is NilmDB or Joule
if source_url is None:
source_node = config_node
nilmdb_source = False
else:
source_node = source_url
await _validate_nilmdb_url(source_node)
nilmdb_source = True
# determine if the destination node is NilmDB or Joule
nilmdb_dest = False
try:
if destination_node is None:
dest_node = config_node
elif type(destination_node) is str:
dest_node = get_node(destination_node)
else:
dest_node = destination_node
except errors.ApiError:
nilmdb_dest = True
dest_node = destination_node
await _validate_nilmdb_url(dest_node)
# retrieve the source stream
src_stream = await _retrieve_source(source_node,
source,
is_nilmdb=nilmdb_source)
# retrieve the destination stream (create it if necessary)
dest_stream = await _retrieve_destination(dest_node,
destination,
src_stream,
is_nilmdb=nilmdb_dest)
# make sure streams are compatible
if src_stream.layout != dest_stream.layout:
raise errors.ApiError(
"Error: source (%s) and destination (%s) datatypes are not compatible"
% (src_stream.layout, dest_stream.layout))
# warn if the elements are not the same
element_warning = False
src_elements = sorted(src_stream.elements, key=attrgetter('index'))
dest_elements = sorted(dest_stream.elements, key=attrgetter('index'))
for i in range(len(src_elements)):
if src_elements[i].name != dest_elements[i].name:
element_warning = True
if src_elements[i].units != dest_elements[i].units:
element_warning = True
if (element_warning and not click.confirm(
"WARNING: Element configurations do not match. Continue?")):
click.echo("Cancelled")
return
# if new is set start and end may not be specified
if new and start is not None:
raise click.ClickException(
"Error: either specify 'new' or a starting timestamp, not both")
# make sure the time bounds make sense
if start is not None:
try:
start = human_to_timestamp(start)
except ValueError:
raise errors.ApiError("invalid start time: [%s]" % start)
if end is not None:
try:
end = human_to_timestamp(end)
except ValueError:
raise errors.ApiError("invalid end time: [%s]" % end)
if (start is not None) and (end is not None) and ((end - start) <= 0):
raise click.ClickException(
"Error: start [%s] must be before end [%s]" %
(datetime.datetime.fromtimestamp(
start / 1e6), datetime.datetime.fromtimestamp(end / 1e6)))
if new:
# pull all the destination intervals and use the end of the last one as the 'end' for the copy
dest_intervals = await _get_intervals(dest_node,
dest_stream,
destination,
None,
None,
is_nilmdb=nilmdb_dest)
if len(dest_intervals) > 0:
start = dest_intervals[-1][-1]
print("Starting copy at [%s]" % timestamp_to_human(start))
else:
print("Starting copy at beginning of source")
# compute the target intervals (source - dest)
src_intervals = await _get_intervals(source_node,
src_stream,
source,
start,
end,
is_nilmdb=nilmdb_source)
dest_intervals = await _get_intervals(dest_node,
dest_stream,
destination,
start,
end,
is_nilmdb=nilmdb_dest)
new_intervals = interval_difference(src_intervals, dest_intervals)
existing_intervals = interval_difference(src_intervals, new_intervals)
async def _copy(intervals):
# compute the duration of data to copy
duration = 0
for interval in intervals:
duration += interval[1] - interval[0]
with click.progressbar(label='Copying data', length=duration) as bar:
for interval in intervals:
await _copy_interval(interval[0], interval[1], bar)
await _copy_annotations(interval[0], interval[1])
async def _copy_annotations(istart, iend):
if nilmdb_source:
src_annotations = await _get_nilmdb_annotations(
source_node, source, istart, iend)
else:
src_annotations = await source_node.annotation_get(src_stream.id,
start=istart,
end=iend)
if nilmdb_dest:
# get *all* the destination annotations, otherwise we'll loose annotations outside this interval
dest_annotations = await _get_nilmdb_annotations(
dest_node, destination)
new_annotations = [
a for a in src_annotations if a not in dest_annotations
]
if len(new_annotations) > 0:
# create ID's for the new annotations
if len(dest_annotations) > 0:
id_val = max([a.id for a in dest_annotations]) + 1
else:
id_val = 0
for a in new_annotations:
a.id = id_val
id_val += 1
await _create_nilmdb_annotations(
dest_node, destination, new_annotations + dest_annotations)
else:
dest_annotations = await dest_node.annotation_get(dest_stream.id,
start=istart,
end=iend)
new_annotations = [
a for a in src_annotations if a not in dest_annotations
]
for annotation in new_annotations:
await dest_node.annotation_create(annotation, dest_stream.id)
if len(new_intervals) == 0:
if len(src_intervals) > 0:
click.echo("Nothing to copy, syncing annotations")
for interval in src_intervals:
await _copy_annotations(interval[0], interval[1])
else:
click.echo("Nothing to copy")
# clean up
if not nilmdb_dest:
await dest_node.close()
if not nilmdb_source:
await source_node.close()
return
async def _copy_interval(istart, iend, bar):
#print("[%s] -> [%s]" % (timestamp_to_human(istart), timestamp_to_human(iend)))
if nilmdb_source:
src_params = {
'path': source,
'binary': 1,
'start': istart,
'end': iend
}
src_url = "{server}/stream/extract".format(server=source_node)
src_headers = {}
src_ssl = None
else:
src_params = {'id': src_stream.id, 'start': istart, 'end': iend}
src_url = "{server}/data".format(server=source_node.session.url)
src_headers = {"X-API-KEY": source_node.session.key}
src_ssl = source_node.session.ssl_context
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(
total=None)) as session:
async with session.get(src_url,
params=src_params,
headers=src_headers,
ssl=src_ssl) as src_response:
if src_response.status != 200:
msg = await src_response.text()
if msg == 'this stream has no data':
# This is not an error because a previous copy may have been interrupted
# This will cause the destination to have an interval gap where the source has no data
# Example: source: |** *******|
# dest: |** | |*******|
# ^--- looks like missing data but there's nothing in the source
return # ignore empty intervals
raise click.ClickException(
"Error reading from source: %s" % msg)
pipe = pipes.InputPipe(stream=dest_stream,
reader=src_response.content)
async def _data_sender():
last_ts = istart
try:
while True:
data = await pipe.read()
pipe.consume(len(data))
if len(data) > 0:
cur_ts = data[-1]['timestamp']
yield data.tobytes()
# total time extents of this chunk
bar.update(cur_ts - last_ts)
last_ts = cur_ts
# if pipe.end_of_interval:
# yield pipes.interval_token(dest_stream.layout). \
# tostring()
except pipes.EmptyPipe:
pass
bar.update(iend - last_ts)
if nilmdb_dest:
dst_params = {
"start": istart,
"end": iend,
"path": destination,
"binary": 1
}
dst_url = "{server}/stream/insert".format(server=dest_node)
await _send_nilmdb_data(
dst_url, dst_params, _data_sender(),
pipes.compute_dtype(dest_stream.layout), session)
else:
dst_url = "{server}/data".format(
server=dest_node.session.url)
dst_params = {"id": dest_stream.id}
dst_headers = {"X-API-KEY": dest_node.session.key}
dst_ssl = dest_node.session.ssl_context
async with session.post(dst_url,
params=dst_params,
data=_data_sender(),
headers=dst_headers,
ssl=dst_ssl,
chunked=True) as dest_response:
if dest_response.status != 200:
msg = await dest_response.text()
raise errors.ApiError(
"Error writing to destination: %s" % msg)
try:
# copy over any new annotations from existing intervals
for interval in existing_intervals:
await _copy_annotations(interval[0], interval[1])
await _copy(new_intervals)
click.echo("\tOK")
# this should be caught by the stream info requests
# it is only generated if the joule server stops during the
# data read/write
except aiohttp.ClientError as e: # pragma: no cover
raise click.ClickException("Error: %s" % str(e))
finally:
if not nilmdb_dest:
await dest_node.close()
if not nilmdb_source:
await source_node.close()
async def _run(loop: asyncio.AbstractEventLoop):
node = api.get_node()
await check_modules(node)
await check_logs(node)
await check_data(node)
await node.close()