def create_dev_app(self) -> web.Application: parser = argparse.ArgumentParser() self._build_args(parser) # must specify module config in JOULE_MODULE_CONFIG environment variable if 'JOULE_MODULE_CONFIG' not in os.environ: raise ConfigurationError( "JOULE_MODULE_CONFIG not set, must specify config file") module_config_file = os.environ['JOULE_MODULE_CONFIG'] if not os.path.isfile(module_config_file): raise ConfigurationError( "JOULE_MODULE_CONFIG is not a module config file") module_args = helpers.load_args_from_file(module_config_file) parsed_args = parser.parse_args(module_args) self.node = api.get_node(parsed_args.node) self.stop_requested = False my_app = self._create_app() async def on_startup(app): app['task'] = await self.run_as_task(parsed_args, app) my_app.on_startup.append(on_startup) async def on_shutdown(app): self.stop() await app['task'] my_app.on_shutdown.append(on_shutdown) return my_app
def test_builds_live_input_pipes(self): server = FakeJoule() src_data = create_source_data(server, is_destination=True) self.start_server(server) loop = asyncio.get_event_loop() my_node = api.get_node() async def runner(): pipes_in, pipes_out = await build_network_pipes( {'input': '/test/source:uint8[x,y,z]'}, {}, {}, my_node, None, None, True) blk1 = await pipes_in['input'].read() self.assertTrue(pipes_in['input'].end_of_interval) pipes_in['input'].consume(len(blk1)) blk2 = await pipes_in['input'].read() pipes_in['input'].consume(len(blk2)) rx_data = np.hstack( (blk1, interval_token(pipes_in['input'].layout), blk2)) np.testing.assert_array_equal(rx_data, src_data) with self.assertRaises(EmptyPipe): await pipes_in['input'].read() await my_node.close() with self.assertLogs(level='INFO') as log: loop.run_until_complete(runner()) log_dump = ' '.join(log.output) self.stop_server() loop.close()
def node(self): # lazy node construction, raise error if it cannot be created if self._node is None: self._node = api.get_node(self.name) click.echo("--connecting to [%s]--" % self._node.name, err=True) self.name = self._node.name return self._node
def test_output_errors(self): server = FakeJoule() create_destination(server) create_source_data(server, is_destination=True) self.start_server(server) loop = asyncio.get_event_loop() my_node = api.get_node() # errors on layout differences with self.assertRaises(ConfigurationError) as e: loop.run_until_complete( build_network_pipes({}, {'output': '/test/dest:float32[x,y,z]'}, {}, my_node, None, None)) self.assertIn('uint8_3', str(e.exception)) # errors if the stream is already being produced with self.assertRaises(ApiError) as e: loop.run_until_complete( build_network_pipes({}, {'output': '/test/source:uint8[e0,e1,e2]'}, {}, my_node, None, None)) loop.run_until_complete(my_node.close()) self.assertIn('already being produced', str(e.exception)) self.stop_server() loop.close()
def test_creates_output_stream_if_necessary(self): server = FakeJoule() self.start_server(server) loop = asyncio.get_event_loop() my_node = api.get_node() self.assertEqual(len(server.streams), 0) async def runner(): # the destination does not exist with self.assertRaises(errors.ApiError): await my_node.data_stream_get("/test/dest") pipes_in, pipes_out = await build_network_pipes( {}, {'output': '/test/dest:uint8[e0,e1,e2]'}, {}, my_node, None, None) await pipes_out['output'].close() # make sure the stream exists dest_stream = await my_node.data_stream_get("/test/dest") self.assertIsNotNone(dest_stream) await my_node.close() with self.assertLogs(level='INFO') as log: loop.run_until_complete(runner()) log_dump = ' '.join(log.output) self.assertIn('creating', log_dump) self.stop_server() loop.close()
def test_input_errors(self): server = FakeJoule() create_source_data(server, is_destination=False) self.start_server(server) loop = asyncio.get_event_loop() my_node = api.get_node() # errors on layout differences with self.assertRaises(ConfigurationError) as e: with self.assertLogs(level='INFO'): loop.run_until_complete( build_network_pipes({'input': '/test/source:uint8[x,y]'}, {}, {}, my_node, 10, 20, force=True)) self.assertIn('uint8_3', str(e.exception)) # errors if live stream is requested but unavailable with self.assertRaises(ApiError) as e: loop.run_until_complete( build_network_pipes({'input': '/test/source:uint8[x,y,z]'}, {}, {}, my_node, None, None)) self.assertIn('not being produced', str(e.exception)) loop.run_until_complete(my_node.close()) self.stop_server() loop.close()
async def setUp(self): self.node1 = api.get_node("node1.joule") followers = await self.node1.follower_list() self.assertEqual(len(followers), 1) self.node2 = followers[0] # add the root user as a follower to node2.joule user = await self.node2.master_add("user", "cli") subprocess.run( f"joule node add node2.joule https://node2.joule:8088 {user.key}". split(" ")) """
def test_configuration_errors(self): server = FakeJoule() loop = asyncio.get_event_loop() self.start_server(server) my_node = api.get_node() # must specify an inline configuration with self.assertRaises(ConfigurationError): loop.run_until_complete( build_network_pipes({'input': '/test/source'}, {}, {}, my_node, 10, 20, force=True)) loop.run_until_complete(my_node.close()) self.stop_server() loop.close()
async def _run(loop: asyncio.AbstractEventLoop): node = api.get_node() procs = start_standalone_procs1() time.sleep(4) # let procs run stop_standalone_procs(procs) time.sleep(8) # close sockets procs = await start_standalone_procs2(node) time.sleep(4) # let procs run stop_standalone_procs(procs) time.sleep(4) # close sockets await check_streams(node) await check_modules(node) await check_data(node) await check_logs(node) await node.close() return
async def setUp(self): self.node = api.get_node() """ └── test ├── f1 │ ├── f1a │ │ ├── s1a1 │ │ └── s1a2 │ └── s1a └── f2 └── f2a └── s2a1a """ stream = api.DataStream(name="s1a1", elements=[api.Element(name="e1")]) await self.node.data_stream_create(stream, "/test/f1/f1a") stream.name = "s1a2" await self.node.data_stream_create(stream, "/test/f1/f1a") stream.name = "s1a" await self.node.data_stream_create(stream, "/test/f1") stream.name = "s2a1a" await self.node.data_stream_create(stream, "/test/f2/f2a")
def node_list(config): """Display authorized nodes.""" try: nodes = api.get_nodes() default_node = api.get_node() except errors.ApiError as e: raise click.ClickException(str(e)) result = [] default_indicator = click.style("\u25CF", fg="green") for node in nodes: if default_node.name == node.name: name = default_indicator + " " + node.name else: name = node.name result.append([name, node.url]) click.echo("List of authorized nodes ("+default_indicator+"=default)") click.echo(tabulate(result, headers=["Node", "URL"], tablefmt="fancy_grid"))
def cli_copy(config: Config, start, end, replace, new, destination_node, source, destination): """Copy events to a different stream.""" try: if destination_node is None: dest_node = config.node else: dest_node = get_node(destination_node) except errors.ApiError: raise click.ClickException( f"Invalid destination node [{destination_node}]") try: asyncio.run( _run(config.node, dest_node, start, end, new, replace, source, destination)) except errors.ApiError as e: raise click.ClickException(str(e)) from e finally: asyncio.run(config.close_node()) if destination_node is not None: # different destination node asyncio.run(dest_node.close()) click.echo("OK")
def test_builds_output_pipes(self): server = FakeJoule() blk1 = helpers.create_data("uint8_3", start=10, step=1, length=10) blk2 = helpers.create_data("uint8_3", start=200, step=1, length=10) create_destination(server) self.start_server(server) my_node = api.get_node() async def runner(): pipes_in, pipes_out = await build_network_pipes( {}, {'output': '/test/dest:uint8[e0,e1,e2]'}, {}, my_node, 10, 210, force=True) await pipes_out['output'].write(blk1) await pipes_out['output'].close_interval() await pipes_out['output'].write(blk2) await pipes_out['output'].close_interval() await pipes_out['output'].close() await my_node.close() asyncio.run(runner()) # first msg should be the data removal (stream_id, start, end) = self.msgs.get() self.assertEqual(int(stream_id), 8) self.assertEqual(int(start), 10) self.assertEqual(int(end), 210) # next message should be the output data mock_entry: MockDbEntry = self.msgs.get() np.testing.assert_array_equal(mock_entry.data[:len(blk1)], blk1) np.testing.assert_array_equal(mock_entry.data[len(blk1):], blk2) self.assertEqual(mock_entry.intervals, [[10, 19], [200, 209]]) self.stop_server()
async def wait_for_follower(): # wait until the local node is online max_tries = 10 num_tries = 0 while num_tries < max_tries: num_tries += 1 sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) result = sock.connect_ex(('localhost', 8088)) sock.close() if result == 0: break time.sleep(0.5) if num_tries == max_tries: print("error, local node is not online") return -1 node1 = api.get_node("node1.joule") while True: followers = await node1.follower_list() if len(followers) == 1: break await asyncio.sleep(0.5) await node1.close() return 0 # success
def test_warns_before_data_removal(self): server = FakeJoule() create_destination(server) self.start_server(server) loop = asyncio.get_event_loop() my_node = api.get_node() async def runner(): with mock.patch('joule.client.helpers.pipes.click') as mock_click: mock_click.confirm = mock.Mock(return_value=False) with self.assertRaises(SystemExit): await build_network_pipes( {}, {'output': '/test/dest:uint8[e0,e1,e2]'}, {}, my_node, 1472500708000000, 1476475108000000, force=False) msg = mock_click.confirm.call_args[0][0] # check for start time self.assertIn("29 Aug", msg) # check for end time self.assertIn("14 Oct", msg) # if end_time not specified anything *after* start is removed mock_click.confirm = mock.Mock(return_value=False) with self.assertRaises(SystemExit): await build_network_pipes( {}, {'output': '/test/dest:uint8[e0,e1,e2]'}, {}, my_node, 1472500708000000, None, force=False) msg = mock_click.confirm.call_args[0][0] # check for start time self.assertIn("29 Aug", msg) # check for *after* self.assertIn("after", msg) # if start_time not specified anything *before* end is removed mock_click.confirm = mock.Mock(return_value=False) with self.assertRaises(SystemExit): await build_network_pipes( {}, {'output': '/test/dest:uint8[e0,e1,e2]'}, {}, my_node, None, 1476475108000000, force=False) msg = mock_click.confirm.call_args[0][0] # check for *before* self.assertIn("before", msg) # check for end time self.assertIn("14 Oct", msg) await my_node.close() # suppress "cancelled" notifications when program exits #with self.assertLogs(level='INFO'): loop.run_until_complete(runner()) self.stop_server() loop.close()
def start(self, parsed_args: argparse.Namespace = None): """ Execute the module. Do not override this function. Creates an event loop and executes the :meth:`run` coroutine. Args: parsed_args: omit to parse the command line arguments .. code-block:: python class ModuleDemo(BaseModule): # body of module... # at a minimum the run coroutine must be implemented if __name__ == "__main__": my_module = ModuleDemo() my_module.start() """ if parsed_args is None: # pragma: no cover parser = argparse.ArgumentParser() self._build_args(parser) module_args = helpers.module_args() parsed_args = parser.parse_args(module_args) # asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) loop = asyncio.get_event_loop() self.stop_requested = False if parsed_args.api_socket != "unset": # should be set by the joule daemon if 'JOULE_CA_FILE' in os.environ: cafile = os.environ['JOULE_CA_FILE'] else: cafile = "" self.node = node.UnixNode("local", parsed_args.api_socket, cafile) else: self.node = api.get_node(parsed_args.node) self.runner: web.AppRunner = loop.run_until_complete( self._start_interface(parsed_args)) app = None if self.runner is not None: app = self.runner.app try: task = loop.run_until_complete(self.run_as_task(parsed_args, app)) except ConfigurationError as e: log.error("ERROR: " + str(e)) self._cleanup(loop) return def stop_task(): # give task no more than 2 seconds to exit loop.call_later(self.STOP_TIMEOUT, task.cancel) # run custom exit routine self.stop() loop.add_signal_handler(signal.SIGINT, stop_task) loop.add_signal_handler(signal.SIGTERM, stop_task) try: loop.run_until_complete(task) except (asyncio.CancelledError, pipes.EmptyPipe, EmptyPipeError) as e: pass finally: self._cleanup(loop)
async def setUp(self): self.node = api.get_node()
async def get_dsn(node_name) -> str: node = api.get_node(node_name) conn_info = await node.db_connection_info() await node.close() return conn_info.to_dsn()
async def _run(config_node, start, end, new, destination_node, source, destination, source_url=None): # determine if the source node is NilmDB or Joule if source_url is None: source_node = config_node nilmdb_source = False else: source_node = source_url await _validate_nilmdb_url(source_node) nilmdb_source = True # determine if the destination node is NilmDB or Joule nilmdb_dest = False try: if destination_node is None: dest_node = config_node elif type(destination_node) is str: dest_node = get_node(destination_node) else: dest_node = destination_node except errors.ApiError: nilmdb_dest = True dest_node = destination_node await _validate_nilmdb_url(dest_node) # retrieve the source stream src_stream = await _retrieve_source(source_node, source, is_nilmdb=nilmdb_source) # retrieve the destination stream (create it if necessary) dest_stream = await _retrieve_destination(dest_node, destination, src_stream, is_nilmdb=nilmdb_dest) # make sure streams are compatible if src_stream.layout != dest_stream.layout: raise errors.ApiError( "Error: source (%s) and destination (%s) datatypes are not compatible" % (src_stream.layout, dest_stream.layout)) # warn if the elements are not the same element_warning = False src_elements = sorted(src_stream.elements, key=attrgetter('index')) dest_elements = sorted(dest_stream.elements, key=attrgetter('index')) for i in range(len(src_elements)): if src_elements[i].name != dest_elements[i].name: element_warning = True if src_elements[i].units != dest_elements[i].units: element_warning = True if (element_warning and not click.confirm( "WARNING: Element configurations do not match. Continue?")): click.echo("Cancelled") return # if new is set start and end may not be specified if new and start is not None: raise click.ClickException( "Error: either specify 'new' or a starting timestamp, not both") # make sure the time bounds make sense if start is not None: try: start = human_to_timestamp(start) except ValueError: raise errors.ApiError("invalid start time: [%s]" % start) if end is not None: try: end = human_to_timestamp(end) except ValueError: raise errors.ApiError("invalid end time: [%s]" % end) if (start is not None) and (end is not None) and ((end - start) <= 0): raise click.ClickException( "Error: start [%s] must be before end [%s]" % (datetime.datetime.fromtimestamp( start / 1e6), datetime.datetime.fromtimestamp(end / 1e6))) if new: # pull all the destination intervals and use the end of the last one as the 'end' for the copy dest_intervals = await _get_intervals(dest_node, dest_stream, destination, None, None, is_nilmdb=nilmdb_dest) if len(dest_intervals) > 0: start = dest_intervals[-1][-1] print("Starting copy at [%s]" % timestamp_to_human(start)) else: print("Starting copy at beginning of source") # compute the target intervals (source - dest) src_intervals = await _get_intervals(source_node, src_stream, source, start, end, is_nilmdb=nilmdb_source) dest_intervals = await _get_intervals(dest_node, dest_stream, destination, start, end, is_nilmdb=nilmdb_dest) new_intervals = interval_difference(src_intervals, dest_intervals) existing_intervals = interval_difference(src_intervals, new_intervals) async def _copy(intervals): # compute the duration of data to copy duration = 0 for interval in intervals: duration += interval[1] - interval[0] with click.progressbar(label='Copying data', length=duration) as bar: for interval in intervals: await _copy_interval(interval[0], interval[1], bar) await _copy_annotations(interval[0], interval[1]) async def _copy_annotations(istart, iend): if nilmdb_source: src_annotations = await _get_nilmdb_annotations( source_node, source, istart, iend) else: src_annotations = await source_node.annotation_get(src_stream.id, start=istart, end=iend) if nilmdb_dest: # get *all* the destination annotations, otherwise we'll loose annotations outside this interval dest_annotations = await _get_nilmdb_annotations( dest_node, destination) new_annotations = [ a for a in src_annotations if a not in dest_annotations ] if len(new_annotations) > 0: # create ID's for the new annotations if len(dest_annotations) > 0: id_val = max([a.id for a in dest_annotations]) + 1 else: id_val = 0 for a in new_annotations: a.id = id_val id_val += 1 await _create_nilmdb_annotations( dest_node, destination, new_annotations + dest_annotations) else: dest_annotations = await dest_node.annotation_get(dest_stream.id, start=istart, end=iend) new_annotations = [ a for a in src_annotations if a not in dest_annotations ] for annotation in new_annotations: await dest_node.annotation_create(annotation, dest_stream.id) if len(new_intervals) == 0: if len(src_intervals) > 0: click.echo("Nothing to copy, syncing annotations") for interval in src_intervals: await _copy_annotations(interval[0], interval[1]) else: click.echo("Nothing to copy") # clean up if not nilmdb_dest: await dest_node.close() if not nilmdb_source: await source_node.close() return async def _copy_interval(istart, iend, bar): #print("[%s] -> [%s]" % (timestamp_to_human(istart), timestamp_to_human(iend))) if nilmdb_source: src_params = { 'path': source, 'binary': 1, 'start': istart, 'end': iend } src_url = "{server}/stream/extract".format(server=source_node) src_headers = {} src_ssl = None else: src_params = {'id': src_stream.id, 'start': istart, 'end': iend} src_url = "{server}/data".format(server=source_node.session.url) src_headers = {"X-API-KEY": source_node.session.key} src_ssl = source_node.session.ssl_context async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout( total=None)) as session: async with session.get(src_url, params=src_params, headers=src_headers, ssl=src_ssl) as src_response: if src_response.status != 200: msg = await src_response.text() if msg == 'this stream has no data': # This is not an error because a previous copy may have been interrupted # This will cause the destination to have an interval gap where the source has no data # Example: source: |** *******| # dest: |** | |*******| # ^--- looks like missing data but there's nothing in the source return # ignore empty intervals raise click.ClickException( "Error reading from source: %s" % msg) pipe = pipes.InputPipe(stream=dest_stream, reader=src_response.content) async def _data_sender(): last_ts = istart try: while True: data = await pipe.read() pipe.consume(len(data)) if len(data) > 0: cur_ts = data[-1]['timestamp'] yield data.tobytes() # total time extents of this chunk bar.update(cur_ts - last_ts) last_ts = cur_ts # if pipe.end_of_interval: # yield pipes.interval_token(dest_stream.layout). \ # tostring() except pipes.EmptyPipe: pass bar.update(iend - last_ts) if nilmdb_dest: dst_params = { "start": istart, "end": iend, "path": destination, "binary": 1 } dst_url = "{server}/stream/insert".format(server=dest_node) await _send_nilmdb_data( dst_url, dst_params, _data_sender(), pipes.compute_dtype(dest_stream.layout), session) else: dst_url = "{server}/data".format( server=dest_node.session.url) dst_params = {"id": dest_stream.id} dst_headers = {"X-API-KEY": dest_node.session.key} dst_ssl = dest_node.session.ssl_context async with session.post(dst_url, params=dst_params, data=_data_sender(), headers=dst_headers, ssl=dst_ssl, chunked=True) as dest_response: if dest_response.status != 200: msg = await dest_response.text() raise errors.ApiError( "Error writing to destination: %s" % msg) try: # copy over any new annotations from existing intervals for interval in existing_intervals: await _copy_annotations(interval[0], interval[1]) await _copy(new_intervals) click.echo("\tOK") # this should be caught by the stream info requests # it is only generated if the joule server stops during the # data read/write except aiohttp.ClientError as e: # pragma: no cover raise click.ClickException("Error: %s" % str(e)) finally: if not nilmdb_dest: await dest_node.close() if not nilmdb_source: await source_node.close()
async def _run(loop: asyncio.AbstractEventLoop): node = api.get_node() await check_modules(node) await check_logs(node) await check_data(node) await node.close()