def test_environ(self):
with LogWriter(prefix='tmp5', note='test_filename_before') as log:
self.assertTrue(Path(log.filename).name.startswith('tmp5'))
os.remove(log.filename)
os.environ['OSGAR_LOGS'] = 'tmp_dir'
with LogWriter(prefix='tmp6', note='test_filename_after') as log:
self.assertEqual(Path(log.filename).parent.name, 'tmp_dir')
os.remove(log.filename)
del os.environ['OSGAR_LOGS']
with self.assertLogs(level=logging.WARNING):
with LogWriter(prefix='tmp7', note='test_filename_after2') as log:
self.assertTrue(Path(log.filename).name.startswith('tmp7'))
os.remove(log.filename)
def test_context_manager(self):
with LogWriter(prefix='tmpp', note='1st test') as log:
self.assertTrue(log.filename.startswith('tmpp'))
filename = log.filename
start_time = log.start_time
t1 = log.write(10, b'\x01\x02\x02\x04')
time.sleep(0.01)
t2 = log.write(10, b'\x05\x06\x07\x08')
self.assertLess(t1, t2)
with LogReader(filename) as log:
self.assertEqual(start_time, log.start_time)
__, stream_id, data = next(log)
self.assertEqual(INFO_STREAM_ID, stream_id)
t, stream_id, data = next(log)
self.assertEqual(stream_id, 10)
self.assertEqual(data, b'\x01\x02\x02\x04')
t, stream_id, data = next(log)
self.assertTrue(t.microseconds > 100)
with self.assertRaises(StopIteration):
__ = next(log)
with LogReader(filename, only_stream_id=10) as log:
for t, stream_id, data in log:
self.assertEqual(stream_id, 10)
self.assertEqual(data, b'\x01\x02\x02\x04')
break
os.remove(log.filename)
def strip_logfile(logfile, only_stream, outfile):
if outfile is None:
outfile = logfile[:-4] + '-strip' + logfile[-4:]
print('Out:', outfile)
with LogReader(logfile, only_stream_id=only_stream) as log:
with LogWriter(filename=outfile, start_time=log.start_time) as out:
for timestamp, stream_id, data in log:
out.write(stream_id=stream_id, data=data, dt=timestamp)
def test_time_overflow(self):
with LogWriter(prefix='tmp8', note='test_time_overflow') as log:
log.start_time = datetime.utcnow() - timedelta(hours=1, minutes=30)
t1 = log.write(1, b'\x01\x02')
self.assertGreater(t1, timedelta(hours=1))
filename = log.filename
with LogReader(filename, only_stream_id=1) as log:
dt, channel, data = next(log)
self.assertGreater(dt, timedelta(minutes=10))
os.remove(filename)
def record(config, log_prefix, log_filename=None, duration_sec=None):
with LogWriter(prefix=log_prefix,
filename=log_filename,
note=str(sys.argv)) as log:
log.write(0, bytes(str(config), 'ascii'))
g_logger.info(log.filename)
with Recorder(config=config['robot'], logger=log) as recorder:
if 'app' in recorder.modules:
app = recorder.modules['app']
app.join(duration_sec)
else:
recorder.stop_requested.wait(duration_sec)
def test_no_eof(self):
with patch('osgar.logger.datetime.datetime'):
osgar.logger.datetime.datetime = TimeStandsStill(
datetime(2019, 1, 1))
with LogWriter(prefix='tmpEof', note='test_EOF') as log:
filename = log.filename
osgar.logger.datetime.datetime = TimeStandsStill(
datetime(2019, 1, 1, 1))
t1 = log.write(1, b'\x01' * 100)
osgar.logger.datetime.datetime = TimeStandsStill(
datetime(2019, 1, 1, 2))
t2 = log.write(1, b'\x02' * 100)
osgar.logger.datetime.datetime = TimeStandsStill(
datetime(2019, 1, 1, 3))
t3 = log.write(1, b'\x03' * 100000)
partial = filename + '.part'
with open(filename, 'rb') as f_in, open(partial, 'wb') as f_out:
f_out.write(f_in.read(100))
f_out.flush()
with LogIndexedReader(partial) as log:
self.assertEqual(len(log), 1)
dt, channel, data = log[0]
with self.assertRaises(IndexError):
log[1]
f_out.write(f_in.read(100))
f_out.flush()
log.grow()
self.assertEqual(len(log), 2)
dt, channel, data = log[1]
self.assertEqual(data, b'\x01' * 100)
self.assertEqual(dt, timedelta(hours=1))
with self.assertRaises(IndexError):
log[2]
f_out.write(f_in.read())
f_out.flush()
log.grow()
dt, channel, data = log[2]
self.assertEqual(dt, timedelta(hours=2))
self.assertEqual(data, b'\x02' * 100)
dt, channel, data = log[3]
self.assertEqual(dt, timedelta(hours=3))
self.assertEqual(len(log), 4)
self.assertEqual(data, b'\x03' * 100000)
os.remove(partial)
os.remove(filename)
def test_read_gen_seek(self):
with LogWriter(prefix='tmpC', note='test_read_gen_seek') as log:
filename = log.filename
t1 = log.write(1, b'\x01')
time.sleep(0.001)
t2 = log.write(1, b'\x02')
time.sleep(0.001)
t3 = log.write(1, b'\x03')
with LogReader(filename, only_stream_id=1) as log:
dt, channel, data = next(log)
self.assertEqual(data, b'\x01')
dt, channel, data = next(log)
self.assertEqual(data, b'\x02')
os.remove(filename)
def test_no_eof(self):
# similar to tail -f (--follow)
with LogWriter(prefix='tmpB', note='test_EOF') as log:
filename = log.filename
t1 = log.write(1, b'\x01'*100)
time.sleep(0.001)
t2 = log.write(1, b'\x02'*100)
time.sleep(0.001)
t3 = log.write(1, b'\x03'*100000)
with LogReader(filename, only_stream_id=1) as log:
dt, channel, data = next(log)
self.assertEqual(data, b'\x01'*100)
dt, channel, data = next(log)
self.assertEqual(data, b'\x02'*100)
partial = filename + '.part'
with open(filename, 'rb') as f_in:
with open(partial, 'wb') as f_out:
f_out.write(f_in.read(100))
with LogReader(partial, only_stream_id=1) as log:
with self.assertLogs(osgar.logger.__name__, logging.ERROR):
with self.assertRaises(StopIteration):
dt, channel, data = next(log)
proc = Timer(0.1, delayed_copy, [filename, partial, 100])
proc.start()
with LogReader(partial, follow=True, only_stream_id=1) as log:
dt, channel, data = next(log)
self.assertEqual(data, b'\x01'*100)
proc.join()
partial = filename + '.part'
with open(filename, 'rb') as f_in:
with open(partial, 'wb') as f_out:
f_out.write(f_in.read(100000))
with LogReader(partial, only_stream_id=1) as log:
dt, channel, data = next(log)
self.assertEqual(data, b'\x01'*100)
dt, channel, data = next(log)
self.assertEqual(data, b'\x02'*100)
with self.assertRaises(AssertionError):
dt, channel, data = next(log)
os.remove(partial)
os.remove(filename)
def test_read_two_streams(self):
with LogWriter(prefix='tmp2', note='test_read_two_streams') as log:
filename = log.filename
t1 = log.write(1, b'\x01\x02\x02\x04')
time.sleep(0.001)
t2 = log.write(3, b'\x05\x06')
time.sleep(0.001)
t3 = log.write(2, b'\x07\x08')
with LogReader(filename, only_stream_id=[1, 2]) as log:
arr = []
for t, stream_id, data in log:
self.assertIn(stream_id, [1, 2])
arr.append((t, stream_id))
self.assertEqual(arr, [(t1, 1), (t3, 2)])
os.remove(log.filename)
def test_indexed_large_block(self):
data = bytes([x for x in range(100)]*1000)
self.assertEqual(len(data), 100000)
with ExitStack() as at_exit:
with LogWriter(prefix='tmpIndexedLarge', note='test_large_block') as log:
at_exit.callback(os.remove, log.filename)
t1 = log.write(1, data)
t2 = log.write(1, data[:0xFFFF])
t3 = log.write(1, b'')
t4 = log.write(1, b'ABC')
t5 = log.write(1, data+data) # multiple split
with LogIndexedReader(log.filename) as log:
self.assertEqual(len(log[1][2]), 100000)
self.assertEqual(len(log[2][2]), 65535)
self.assertEqual(len(log[3][2]), 0)
self.assertEqual(len(log[4][2]), 3)
self.assertEqual(len(log[5][2]), 200000)
def test_large_block(self):
data = bytes([x for x in range(100)] * 1000)
self.assertEqual(len(data), 100000)
with LogWriter(prefix='tmp4', note='test_large_block') as log:
filename = log.filename
t1 = log.write(1, data)
t2 = log.write(1, data[:0xFFFF])
t3 = log.write(1, b'')
t4 = log.write(1, b'ABC')
t5 = log.write(1, data + data) # multiple split
with LogReader(filename, only_stream_id=1) as log:
arr = []
for __, __, data in log:
arr.append(data)
self.assertEqual([len(x) for x in arr],
[100000, 65535, 0, 3, 200000])
os.remove(log.filename)
def test_register(self):
with LogWriter(prefix='tmp2', note='test_register') as log:
filename = log.filename
self.assertEqual(log.register('raw'), 1)
with self.assertRaises(AssertionError):
log.register('raw') # duplicity name
self.assertEqual(log.register('gps.position'), 2)
with LogReader(filename, only_stream_id=INFO_STREAM_ID) as log:
arr = []
for __, __, data in log:
if b'names' in data:
arr.append(data)
self.assertEqual(len(arr), 2, arr)
self.assertEqual(arr[0], b"{'names': ['raw']}")
self.assertEqual(arr[1], b"{'names': ['raw', 'gps.position']}")
os.remove(filename)
def test_log_asserter(self):
with LogWriter(prefix='tmp3', note='test_log_asserter') as log:
filename = log.filename
t1 = log.write(1, b'\x01\x02')
time.sleep(0.001)
t2 = log.write(2, b'\x05')
time.sleep(0.001)
t3 = log.write(1, b'\x07\x08')
with LogAsserter(filename, only_stream_id=[1, 2]) as log:
log.assert_stream_id = 2
arr = []
for t, stream_id, data in log:
self.assertIn(stream_id, [1, 2])
if stream_id == 2:
log.write(2, b'\x05')
arr.append((t, stream_id))
self.assertEqual(arr, [(t1, 1), (t2, 2), (t3, 1)])
os.remove(log.filename)
def main():
import argparse
from osgar.lib.config import load as config_load
from osgar.record import Recorder
from osgar.logger import LogWriter, LogReader
parser = argparse.ArgumentParser(description='SubT Challenge')
subparsers = parser.add_subparsers(help='sub-command help', dest='command')
subparsers.required = True
parser_run = subparsers.add_parser('run', help='run on real HW')
parser_run.add_argument('config', nargs='+', help='configuration file')
parser_run.add_argument('--note', help='add description')
parser_replay = subparsers.add_parser('replay', help='replay from logfile')
parser_replay.add_argument('logfile', help='recorded log file')
parser_replay.add_argument('--force', '-F', dest='force', action='store_true', help='force replay even for failing output asserts')
parser_replay.add_argument('--config', nargs='+', help='force alternative configuration file')
args = parser.parse_args()
if args.command == 'replay':
from osgar.replay import replay
args.module = 'app'
game = replay(args, application=SubTChallenge)
game.play()
elif args.command == 'run':
# To reduce latency spikes as described in https://morepypy.blogspot.com/2019/01/pypy-for-low-latency-systems.html.
# Increased latency leads to uncontrolled behavior and robot either missing turns or hitting walls.
# Disabled garbage collection needs to be paired with gc.collect() at place(s) that are not time sensitive.
gc.disable()
# support simultaneously multiple platforms
prefix = os.path.basename(args.config[0]).split('.')[0] + '-'
log = LogWriter(prefix=prefix, note=str(sys.argv))
config = config_load(*args.config)
log.write(0, bytes(str(config), 'ascii')) # write configuration
robot = Recorder(config=config['robot'], logger=log, application=SubTChallenge)
game = robot.modules['app'] # TODO nicer reference
robot.start()
game.play()
robot.finish()
def test_indexed_reader(self):
note = 'test_indexed_reader'
sample = [b'\x01', b'\x02', b'\x03']
times = []
with ExitStack() as at_exit:
with LogWriter(prefix='tmpIndexed', note=note) as log:
at_exit.callback(os.remove, log.filename)
for a in sample:
t = log.write(1, a)
times.append(t)
time.sleep(0.001)
with LogIndexedReader(log.filename) as log:
dt, channel, data = log[0]
assert data.decode('utf-8') == note, data
for i in range(len(sample)):
dt, channel, data = log[i + 1]
assert data == sample[i], data
assert dt == times[i], (dt, times[i])
assert len(log) == len(sample) + 1, len(log)
def record(config_filename, log_prefix, duration_sec=None, application=None):
log = LogWriter(prefix=log_prefix, note=str(sys.argv))
if type(config_filename) == str:
config = load(config_filename)
else:
config = load(*config_filename)
log.write(0, bytes(str(config), 'ascii')) # write configuration
recorder = Recorder(config=config['robot'],
logger=log,
application=application)
recorder.start()
if application is not None:
game = recorder.modules['app'] # TODO nicer reference
game.join() # wait for application termination
else:
if duration_sec is None:
while True:
time.sleep(1.0)
else:
time.sleep(duration_sec)
recorder.finish()
try:
# https://github.com/mesonbuild/meson/issues/4087
# without timeout the call can hang the process forever
with urllib.request.urlopen(self.url, timeout=0.5) as f:
data = f.read()
if len(data) > 0:
self.bus.publish('raw', data)
except socket.timeout:
pass
if self.sleep is not None:
self.bus.sleep(self.sleep)
def request_stop(self):
self.bus.shutdown()
if __name__ == "__main__":
import time
from osgar.bus import Bus
config = {'host': 'localhost', 'port': 8001, 'timeout': 1.0}
with LogWriter(prefix='test-tcp-') as log:
bus = Bus(log)
device = LogTCPStaticIP(config, bus=bus.handle('tcp'))
device.start()
time.sleep(2)
device.request_stop()
device.join()
# vim: expandtab sw=4 ts=4
nargs='+',
help='force alternative configuration file')
args = parser.parse_args()
if args.command == 'replay':
from osgar.replay import replay
args.module = 'app'
game = replay(args, application=SubTChallenge)
game.play()
elif args.command == 'run':
# To reduce latency spikes as described in https://morepypy.blogspot.com/2019/01/pypy-for-low-latency-systems.html.
# Increased latency leads to uncontrolled behavior and robot either missing turns or hitting walls.
# Disabled garbage collection needs to be paired with gc.collect() at place(s) that are not time sensitive.
gc.disable()
# support simultaneously multiple platforms
prefix = os.path.basename(args.config[0]).split('.')[0] + '-'
log = LogWriter(prefix=prefix, note=str(sys.argv))
config = config_load(*args.config)
log.write(0, bytes(str(config), 'ascii')) # write configuration
robot = Recorder(config=config['robot'],
logger=log,
application=SubTChallenge)
game = robot.modules['app'] # TODO nicer reference
robot.start()
game.play()
robot.finish()
# vim: expandtab sw=4 ts=4
self.bus.publish('raw', data)
def slot_raw(self, data):
self.com.write(data)
def run_output(self):
try:
while True:
__, __, data = self.bus.listen()
self.slot_raw(data)
except BusShutdownException:
pass
def request_stop(self):
self.bus.shutdown()
if __name__ == "__main__":
import time
from osgar.bus import BusHandler
config = {'port': 'COM5', 'speed': 4800}
log = LogWriter(prefix='test-')
device = LogSerial(config, bus=BusHandler(log, out={'raw': []}))
device.start()
time.sleep(2)
device.request_stop()
device.join()
# vim: expandtab sw=4 ts=4
'-F',
dest='force',
action='store_true',
help='force replay even for failing output asserts')
parser_replay.add_argument('--config',
nargs='+',
help='force alternative configuration file')
args = parser.parse_args()
if args.command == 'replay':
from replay import replay
args.module = 'app'
game = replay(args, application=RoboOrienteering2018)
game.play()
elif args.command == 'run':
log = LogWriter(prefix='ro2018-', note=str(sys.argv))
config = config_load(*args.config)
log.write(0, bytes(str(config), 'ascii')) # write configuration
recorder = Recorder(config=config['robot'],
logger=log,
application=RoboOrienteering2018)
game = robot.modules['app'] # TODO nicer reference
robot.start()
game.play()
recorder.finish()
else:
assert False, args.command # unsupported command
# vim: expandtab sw=4 ts=4
def test_writer_prefix(self):
log = LogWriter(prefix='tmp')
self.assertTrue(log.filename.startswith('tmp'))
log.close()
os.remove(log.filename)
stream_names = lookup_stream_names(args.logfile)
def get_camera_name_id(camera_pose):
x, y = camera_pose[0][:2]
if y == 0:
camera_direction = 'front' if x > 0 else 'rear'
elif y < 0:
camera_direction = 'right'
else: # y > 0
camera_direction = 'left'
return stream_names.index(f'fromrospy.rgbd_{camera_direction}') + 1
poses = []
crash_time = None
with LogReader(args.logfile,
only_stream_id=[pose3d_stream_id, crash_stream_id]) as logreader, LogWriter(
filename=outfile, start_time=logreader.start_time) as f:
for name in stream_names:
f.register(name)
for time, stream, raw_bytes in logreader:
data = deserialize(raw_bytes)
if stream == pose3d_stream_id:
poses.append(data)
crash_time = None
elif stream == crash_stream_id:
robot_pose, camera_pose, rgb_compressed, depth_compressed = data
if crash_time is None:
crash_time = time
if args.verbose:
print(f'------------- {crash_time} -------------')
if args.verbose:
print(nearest_pose(poses, robot_pose), camera_pose[0])
subparsers.required = True
parser_run = subparsers.add_parser('run', help='run on real HW')
parser_run.add_argument('config', nargs='+', help='configuration file')
parser_run.add_argument('--note', help='add description')
parser_replay = subparsers.add_parser('replay', help='replay from logfile')
parser_replay.add_argument('logfile', help='recorded log file')
parser_replay.add_argument('--force', '-F', dest='force', action='store_true', help='force replay even for failing output asserts')
parser_replay.add_argument('--config', nargs='+', help='force alternative configuration file')
parser_replay.add_argument('--verbose', '-v', help="verbose mode", action='store_true')
args = parser.parse_args()
if args.command == 'replay':
from replay import replay
args.module = 'app'
game = replay(args, application=SICKRobot2018)
game.verbose = args.verbose
game.play()
elif args.command == 'run':
log = LogWriter(prefix='eduro-', note=str(sys.argv))
config = config_load(*args.config)
log.write(0, bytes(str(config), 'ascii')) # write configuration
robot = Recorder(config=config['robot'], logger=log, application=SICKRobot2018)
game = robot.modules['app'] # TODO nicer reference
robot.start()
game.play()
robot.finish()
# vim: expandtab sw=4 ts=4
def test_writer_prefix(self):
with LogWriter(prefix='tmp') as log:
self.assertTrue(Path(log.filename).name.startswith('tmp'))
os.remove(log.filename)
