def test_environ(self): with LogWriter(prefix='tmp5', note='test_filename_before') as log: self.assertTrue(log.filename.startswith('tmp5')) os.remove(log.filename) os.environ['OSGAR_LOGS'] = 'tmp_dir' with LogWriter(prefix='tmp6', note='test_filename_after') as log: self.assertTrue(log.filename.startswith('tmp_dir')) os.remove(log.filename) del os.environ['OSGAR_LOGS'] with LogWriter(prefix='tmp7', note='test_filename_after2') as log: self.assertTrue(log.filename.startswith('tmp7')) os.remove(log.filename)
def test_environ(self): with LogWriter(prefix='tmp5', note='test_filename_before') as log: self.assertTrue(Path(log.filename).name.startswith('tmp5')) os.remove(log.filename) os.environ['OSGAR_LOGS'] = 'tmp_dir' with LogWriter(prefix='tmp6', note='test_filename_after') as log: self.assertEqual(Path(log.filename).parent.name, 'tmp_dir') os.remove(log.filename) del os.environ['OSGAR_LOGS'] with self.assertLogs(level=logging.WARNING): with LogWriter(prefix='tmp7', note='test_filename_after2') as log: self.assertTrue(Path(log.filename).name.startswith('tmp7')) os.remove(log.filename)
def test_context_manager(self): with LogWriter(prefix='tmpp', note='1st test') as log: self.assertTrue(log.filename.startswith('tmpp')) filename = log.filename start_time = log.start_time t1 = log.write(10, b'\x01\x02\x02\x04') time.sleep(0.01) t2 = log.write(10, b'\x05\x06\x07\x08') self.assertLess(t1, t2) with LogReader(filename) as log: self.assertEqual(start_time, log.start_time) __, stream_id, data = next(log) self.assertEqual(INFO_STREAM_ID, stream_id) t, stream_id, data = next(log) self.assertEqual(stream_id, 10) self.assertEqual(data, b'\x01\x02\x02\x04') t, stream_id, data = next(log) self.assertTrue(t.microseconds > 100) with self.assertRaises(StopIteration): __ = next(log) with LogReader(filename, only_stream_id=10) as log: for t, stream_id, data in log: self.assertEqual(stream_id, 10) self.assertEqual(data, b'\x01\x02\x02\x04') break os.remove(log.filename)
def strip_logfile(logfile, only_stream, outfile): if outfile is None: outfile = logfile[:-4] + '-strip' + logfile[-4:] print('Out:', outfile) with LogReader(logfile, only_stream_id=only_stream) as log: with LogWriter(filename=outfile, start_time=log.start_time) as out: for timestamp, stream_id, data in log: out.write(stream_id=stream_id, data=data, dt=timestamp)
def test_time_overflow(self): with LogWriter(prefix='tmp8', note='test_time_overflow') as log: log.start_time = datetime.utcnow() - timedelta(hours=1, minutes=30) t1 = log.write(1, b'\x01\x02') self.assertGreater(t1, timedelta(hours=1)) filename = log.filename with LogReader(filename, only_stream_id=1) as log: dt, channel, data = next(log) self.assertGreater(dt, timedelta(minutes=10)) os.remove(filename)
def main(): import argparse from osgar.lib.config import load as config_load from osgar.record import Recorder from osgar.logger import LogWriter, LogReader parser = argparse.ArgumentParser(description='SubT Challenge') subparsers = parser.add_subparsers(help='sub-command help', dest='command') subparsers.required = True parser_run = subparsers.add_parser('run', help='run on real HW') parser_run.add_argument('config', nargs='+', help='configuration file') parser_run.add_argument('--note', help='add description') parser_replay = subparsers.add_parser('replay', help='replay from logfile') parser_replay.add_argument('logfile', help='recorded log file') parser_replay.add_argument('--force', '-F', dest='force', action='store_true', help='force replay even for failing output asserts') parser_replay.add_argument('--config', nargs='+', help='force alternative configuration file') args = parser.parse_args() if args.command == 'replay': from osgar.replay import replay args.module = 'app' game = replay(args, application=SubTChallenge) game.play() elif args.command == 'run': # To reduce latency spikes as described in https://morepypy.blogspot.com/2019/01/pypy-for-low-latency-systems.html. # Increased latency leads to uncontrolled behavior and robot either missing turns or hitting walls. # Disabled garbage collection needs to be paired with gc.collect() at place(s) that are not time sensitive. gc.disable() # support simultaneously multiple platforms prefix = os.path.basename(args.config[0]).split('.')[0] + '-' log = LogWriter(prefix=prefix, note=str(sys.argv)) config = config_load(*args.config) log.write(0, bytes(str(config), 'ascii')) # write configuration robot = Recorder(config=config['robot'], logger=log, application=SubTChallenge) game = robot.modules['app'] # TODO nicer reference robot.start() game.play() robot.finish()
def record(config, log_prefix, log_filename=None, duration_sec=None): with LogWriter(prefix=log_prefix, filename=log_filename, note=str(sys.argv)) as log: log.write(0, bytes(str(config), 'ascii')) g_logger.info(log.filename) with Recorder(config=config['robot'], logger=log) as recorder: if 'app' in recorder.modules: app = recorder.modules['app'] app.join(duration_sec) else: recorder.stop_requested.wait(duration_sec)
def record(config_filename, log_prefix, duration_sec=None, application=None): log = LogWriter(prefix=log_prefix, note=str(sys.argv)) if type(config_filename) == str: config = load(config_filename) else: config = load(*config_filename) log.write(0, bytes(str(config), 'ascii')) # write configuration recorder = Recorder(config=config['robot'], logger=log, application=application) recorder.start() if application is not None: game = recorder.modules['app'] # TODO nicer reference game.join() # wait for application termination else: if duration_sec is None: while True: time.sleep(1.0) else: time.sleep(duration_sec) recorder.finish()
def test_no_eof(self): with patch('osgar.logger.datetime.datetime'): osgar.logger.datetime.datetime = TimeStandsStill( datetime(2019, 1, 1)) with LogWriter(prefix='tmpEof', note='test_EOF') as log: filename = log.filename osgar.logger.datetime.datetime = TimeStandsStill( datetime(2019, 1, 1, 1)) t1 = log.write(1, b'\x01' * 100) osgar.logger.datetime.datetime = TimeStandsStill( datetime(2019, 1, 1, 2)) t2 = log.write(1, b'\x02' * 100) osgar.logger.datetime.datetime = TimeStandsStill( datetime(2019, 1, 1, 3)) t3 = log.write(1, b'\x03' * 100000) partial = filename + '.part' with open(filename, 'rb') as f_in, open(partial, 'wb') as f_out: f_out.write(f_in.read(100)) f_out.flush() with LogIndexedReader(partial) as log: self.assertEqual(len(log), 1) dt, channel, data = log[0] with self.assertRaises(IndexError): log[1] f_out.write(f_in.read(100)) f_out.flush() log.grow() self.assertEqual(len(log), 2) dt, channel, data = log[1] self.assertEqual(data, b'\x01' * 100) self.assertEqual(dt, timedelta(hours=1)) with self.assertRaises(IndexError): log[2] f_out.write(f_in.read()) f_out.flush() log.grow() dt, channel, data = log[2] self.assertEqual(dt, timedelta(hours=2)) self.assertEqual(data, b'\x02' * 100) dt, channel, data = log[3] self.assertEqual(dt, timedelta(hours=3)) self.assertEqual(len(log), 4) self.assertEqual(data, b'\x03' * 100000) os.remove(partial) os.remove(filename)
def test_read_gen_seek(self): with LogWriter(prefix='tmpC', note='test_read_gen_seek') as log: filename = log.filename t1 = log.write(1, b'\x01') time.sleep(0.001) t2 = log.write(1, b'\x02') time.sleep(0.001) t3 = log.write(1, b'\x03') with LogReader(filename, only_stream_id=1) as log: dt, channel, data = next(log) self.assertEqual(data, b'\x01') dt, channel, data = next(log) self.assertEqual(data, b'\x02') os.remove(filename)
def test_no_eof(self): # similar to tail -f (--follow) with LogWriter(prefix='tmpB', note='test_EOF') as log: filename = log.filename t1 = log.write(1, b'\x01'*100) time.sleep(0.001) t2 = log.write(1, b'\x02'*100) time.sleep(0.001) t3 = log.write(1, b'\x03'*100000) with LogReader(filename, only_stream_id=1) as log: dt, channel, data = next(log) self.assertEqual(data, b'\x01'*100) dt, channel, data = next(log) self.assertEqual(data, b'\x02'*100) partial = filename + '.part' with open(filename, 'rb') as f_in: with open(partial, 'wb') as f_out: f_out.write(f_in.read(100)) with LogReader(partial, only_stream_id=1) as log: with self.assertLogs(osgar.logger.__name__, logging.ERROR): with self.assertRaises(StopIteration): dt, channel, data = next(log) proc = Timer(0.1, delayed_copy, [filename, partial, 100]) proc.start() with LogReader(partial, follow=True, only_stream_id=1) as log: dt, channel, data = next(log) self.assertEqual(data, b'\x01'*100) proc.join() partial = filename + '.part' with open(filename, 'rb') as f_in: with open(partial, 'wb') as f_out: f_out.write(f_in.read(100000)) with LogReader(partial, only_stream_id=1) as log: dt, channel, data = next(log) self.assertEqual(data, b'\x01'*100) dt, channel, data = next(log) self.assertEqual(data, b'\x02'*100) with self.assertRaises(AssertionError): dt, channel, data = next(log) os.remove(partial) os.remove(filename)
def test_read_two_streams(self): with LogWriter(prefix='tmp2', note='test_read_two_streams') as log: filename = log.filename t1 = log.write(1, b'\x01\x02\x02\x04') time.sleep(0.001) t2 = log.write(3, b'\x05\x06') time.sleep(0.001) t3 = log.write(2, b'\x07\x08') with LogReader(filename, only_stream_id=[1, 2]) as log: arr = [] for t, stream_id, data in log: self.assertIn(stream_id, [1, 2]) arr.append((t, stream_id)) self.assertEqual(arr, [(t1, 1), (t3, 2)]) os.remove(log.filename)
def test_indexed_large_block(self): data = bytes([x for x in range(100)]*1000) self.assertEqual(len(data), 100000) with ExitStack() as at_exit: with LogWriter(prefix='tmpIndexedLarge', note='test_large_block') as log: at_exit.callback(os.remove, log.filename) t1 = log.write(1, data) t2 = log.write(1, data[:0xFFFF]) t3 = log.write(1, b'') t4 = log.write(1, b'ABC') t5 = log.write(1, data+data) # multiple split with LogIndexedReader(log.filename) as log: self.assertEqual(len(log[1][2]), 100000) self.assertEqual(len(log[2][2]), 65535) self.assertEqual(len(log[3][2]), 0) self.assertEqual(len(log[4][2]), 3) self.assertEqual(len(log[5][2]), 200000)
def test_large_block(self): data = bytes([x for x in range(100)] * 1000) self.assertEqual(len(data), 100000) with LogWriter(prefix='tmp4', note='test_large_block') as log: filename = log.filename t1 = log.write(1, data) t2 = log.write(1, data[:0xFFFF]) t3 = log.write(1, b'') t4 = log.write(1, b'ABC') t5 = log.write(1, data + data) # multiple split with LogReader(filename, only_stream_id=1) as log: arr = [] for __, __, data in log: arr.append(data) self.assertEqual([len(x) for x in arr], [100000, 65535, 0, 3, 200000]) os.remove(log.filename)
def test_register(self): with LogWriter(prefix='tmp2', note='test_register') as log: filename = log.filename self.assertEqual(log.register('raw'), 1) with self.assertRaises(AssertionError): log.register('raw') # duplicity name self.assertEqual(log.register('gps.position'), 2) with LogReader(filename, only_stream_id=INFO_STREAM_ID) as log: arr = [] for __, __, data in log: if b'names' in data: arr.append(data) self.assertEqual(len(arr), 2, arr) self.assertEqual(arr[0], b"{'names': ['raw']}") self.assertEqual(arr[1], b"{'names': ['raw', 'gps.position']}") os.remove(filename)
def test_log_asserter(self): with LogWriter(prefix='tmp3', note='test_log_asserter') as log: filename = log.filename t1 = log.write(1, b'\x01\x02') time.sleep(0.001) t2 = log.write(2, b'\x05') time.sleep(0.001) t3 = log.write(1, b'\x07\x08') with LogAsserter(filename, only_stream_id=[1, 2]) as log: log.assert_stream_id = 2 arr = [] for t, stream_id, data in log: self.assertIn(stream_id, [1, 2]) if stream_id == 2: log.write(2, b'\x05') arr.append((t, stream_id)) self.assertEqual(arr, [(t1, 1), (t2, 2), (t3, 1)]) os.remove(log.filename)
def test_indexed_reader(self): note = 'test_indexed_reader' sample = [b'\x01', b'\x02', b'\x03'] times = [] with ExitStack() as at_exit: with LogWriter(prefix='tmpIndexed', note=note) as log: at_exit.callback(os.remove, log.filename) for a in sample: t = log.write(1, a) times.append(t) time.sleep(0.001) with LogIndexedReader(log.filename) as log: dt, channel, data = log[0] assert data.decode('utf-8') == note, data for i in range(len(sample)): dt, channel, data = log[i + 1] assert data == sample[i], data assert dt == times[i], (dt, times[i]) assert len(log) == len(sample) + 1, len(log)
try: # https://github.com/mesonbuild/meson/issues/4087 # without timeout the call can hang the process forever with urllib.request.urlopen(self.url, timeout=0.5) as f: data = f.read() if len(data) > 0: self.bus.publish('raw', data) except socket.timeout: pass if self.sleep is not None: self.bus.sleep(self.sleep) def request_stop(self): self.bus.shutdown() if __name__ == "__main__": import time from osgar.bus import Bus config = {'host': 'localhost', 'port': 8001, 'timeout': 1.0} with LogWriter(prefix='test-tcp-') as log: bus = Bus(log) device = LogTCPStaticIP(config, bus=bus.handle('tcp')) device.start() time.sleep(2) device.request_stop() device.join() # vim: expandtab sw=4 ts=4
stream_names = lookup_stream_names(args.logfile) def get_camera_name_id(camera_pose): x, y = camera_pose[0][:2] if y == 0: camera_direction = 'front' if x > 0 else 'rear' elif y < 0: camera_direction = 'right' else: # y > 0 camera_direction = 'left' return stream_names.index(f'fromrospy.rgbd_{camera_direction}') + 1 poses = [] crash_time = None with LogReader(args.logfile, only_stream_id=[pose3d_stream_id, crash_stream_id]) as logreader, LogWriter( filename=outfile, start_time=logreader.start_time) as f: for name in stream_names: f.register(name) for time, stream, raw_bytes in logreader: data = deserialize(raw_bytes) if stream == pose3d_stream_id: poses.append(data) crash_time = None elif stream == crash_stream_id: robot_pose, camera_pose, rgb_compressed, depth_compressed = data if crash_time is None: crash_time = time if args.verbose: print(f'------------- {crash_time} -------------') if args.verbose: print(nearest_pose(poses, robot_pose), camera_pose[0])
nargs='+', help='force alternative configuration file') args = parser.parse_args() if args.command == 'replay': from osgar.replay import replay args.module = 'app' game = replay(args, application=SubTChallenge) game.play() elif args.command == 'run': # To reduce latency spikes as described in https://morepypy.blogspot.com/2019/01/pypy-for-low-latency-systems.html. # Increased latency leads to uncontrolled behavior and robot either missing turns or hitting walls. # Disabled garbage collection needs to be paired with gc.collect() at place(s) that are not time sensitive. gc.disable() # support simultaneously multiple platforms prefix = os.path.basename(args.config[0]).split('.')[0] + '-' log = LogWriter(prefix=prefix, note=str(sys.argv)) config = config_load(*args.config) log.write(0, bytes(str(config), 'ascii')) # write configuration robot = Recorder(config=config['robot'], logger=log, application=SubTChallenge) game = robot.modules['app'] # TODO nicer reference robot.start() game.play() robot.finish() # vim: expandtab sw=4 ts=4
self.bus.publish('raw', data) def slot_raw(self, data): self.com.write(data) def run_output(self): try: while True: __, __, data = self.bus.listen() self.slot_raw(data) except BusShutdownException: pass def request_stop(self): self.bus.shutdown() if __name__ == "__main__": import time from osgar.bus import BusHandler config = {'port': 'COM5', 'speed': 4800} log = LogWriter(prefix='test-') device = LogSerial(config, bus=BusHandler(log, out={'raw': []})) device.start() time.sleep(2) device.request_stop() device.join() # vim: expandtab sw=4 ts=4
subparsers.required = True parser_run = subparsers.add_parser('run', help='run on real HW') parser_run.add_argument('config', nargs='+', help='configuration file') parser_run.add_argument('--note', help='add description') parser_replay = subparsers.add_parser('replay', help='replay from logfile') parser_replay.add_argument('logfile', help='recorded log file') parser_replay.add_argument('--force', '-F', dest='force', action='store_true', help='force replay even for failing output asserts') parser_replay.add_argument('--config', nargs='+', help='force alternative configuration file') parser_replay.add_argument('--verbose', '-v', help="verbose mode", action='store_true') args = parser.parse_args() if args.command == 'replay': from replay import replay args.module = 'app' game = replay(args, application=SICKRobot2018) game.verbose = args.verbose game.play() elif args.command == 'run': log = LogWriter(prefix='eduro-', note=str(sys.argv)) config = config_load(*args.config) log.write(0, bytes(str(config), 'ascii')) # write configuration robot = Recorder(config=config['robot'], logger=log, application=SICKRobot2018) game = robot.modules['app'] # TODO nicer reference robot.start() game.play() robot.finish() # vim: expandtab sw=4 ts=4
def test_writer_prefix(self): log = LogWriter(prefix='tmp') self.assertTrue(log.filename.startswith('tmp')) log.close() os.remove(log.filename)
def test_writer_prefix(self): with LogWriter(prefix='tmp') as log: self.assertTrue(Path(log.filename).name.startswith('tmp')) os.remove(log.filename)