async def test_save_no_routes_before_and_after(self): repo = MemoryRepo() writer = TreeWriter(repo) app, settings = get_test_app_and_settings(repo) event = Event(app, 'test_event', {'title': 'Test event'}, []) await event.save('save test_event', tree_writer=writer, save_routes=True) self.assertFalse(writer.exists('events/test_event/routes'))
def test_rm_non_existant(self): repo = MemoryRepo() writer = TreeWriter(repo) writer.set_data('a/b', b'b') with self.assertRaises(KeyError): writer.remove('a/c') writer.set_data('a/d', b'd')
async def assign_rider_colors(app, settings, args): event_name = event_name_clean(args.event_name, settings) tree_writer = TreeWriter(app['trackers.data_repo']) event = await trackers.events.Event.load(app, event_name, tree_writer) assign_rider_colors_inner(event) await event.save(f'{event_name}: assign_rider_colors', tree_writer=tree_writer)
async def load_riders_from_csv(app, settings, args): event_name = event_name_clean(args.event_name, settings) tree_writer = TreeWriter(app['trackers.data_repo']) event = await trackers.events.Event.load(app, event_name, tree_writer) import csv reader = csv.DictReader(sys.stdin) def trackers_from_row(row): if row.get('Traccar Device Id'): yield {'type': 'traccar', 'unique_id': row['Traccar Device Id']} if row.get('TKStorage'): yield {'type': 'tkstorage', 'id': row['TKStorage']} if row.get('Spot'): yield {'type': 'spot', 'feed_id': row['Spot']} event.config['riders'] = [{ 'name': row['Name'], 'name_short': row['Short Name'], 'trackers': list(trackers_from_row(row)), } for row in reader] assign_rider_colors_inner(event) await event.save(f'{event_name}: load_riders_from_csv', tree_writer=tree_writer)
async def convert_to_static(app, settings, args): tree_writer = TreeWriter(app['trackers.data_repo']) event_name = event_name_clean(args.event_name, settings) event = await trackers.events.Event.load(app, event_name, tree_writer) await event.convert_to_static(tree_writer) await event.store_analyse(tree_writer)
async def test_from_load_with_routes_old(self): repo = MemoryRepo() writer = TreeWriter(repo) writer.set_data('events/test_event/data.yaml', '{}'.encode()) writer.set_data('events/test_event/routes', b'\x91\x81\xa6points\x90') writer.commit('add test_event') app, settings = get_test_app_and_settings(repo) event = await Event.load(app, 'test_event', writer) self.assertEqual(event.routes, [{'points': []}])
async def test_save_no_routes(self): repo = MemoryRepo() writer = TreeWriter(repo) writer.set_data('events/test_event/data.yaml', '{}'.encode()) writer.set_data('events/test_event/routes', b'\x91\x81\xa6points\x90') writer.commit('add test_event') app, settings = get_test_app_and_settings(repo) event = await Event.load(app, 'test_event', writer) event.routes.pop() await event.save('save test_event', tree_writer=writer, save_routes=True) self.assertFalse(writer.exists('events/test_event/routes'))
async def test_start_json(self): repo = MemoryRepo() writer = TreeWriter(repo) writer.set_data('events/test_event/data.yaml', '{}'.encode()) writer.set_data('events/test_event/test_rider', '[]'.encode()) writer.commit('add test_event') app, settings = get_test_app_and_settings(repo) event = await Event.load(app, 'test_event', writer) tracker = await static_start_event_tracker(app, event, 'Test rider', {'name': 'test_rider', 'format': 'json'}, None, None) await tracker.complete() self.assertEqual(tracker.points, [])
async def reformat_event(app, settings, args): event_name = event_name_clean(args.event_name, settings) writer = TreeWriter(app['trackers.data_repo']) event = await trackers.events.Event.load(app, event_name, writer) for route in event.routes: for key in ('start_distance', 'end_distance', 'dist_factor'): if key in route: route[key] = float(route[key]) await event.save(f'{event_name}: reformat_event', tree_writer=writer, save_routes=True)
async def process_event_routes(app, settings, args): event_name = event_name_clean(args.event_name, settings) writer = TreeWriter(app['trackers.data_repo']) event = await trackers.events.Event.load(app, event_name, writer) for route in event.routes: if args.rdp_epsilon: route['rdp_epsilon'] = args.rdp_epsilon await process_route(settings, route) process_secondary_route_details(event.routes) await event.save(f'{event_name}: process_event_routes', tree_writer=writer, save_routes=True)
async def test_save(self): repo = MemoryRepo() writer = TreeWriter(repo) app, settings = get_test_app_and_settings(repo) event = Event(app, 'test_event', {'title': 'Test event'}, [{ 'title': 'foobar', 'points': [] }]) await event.save('save test_event', tree_writer=writer, save_routes=True) self.assertEqual( writer.get('events/test_event/data.yaml').data.decode(), 'title: Test event\n') self.assertEqual( writer.get('events/test_event/routes.yaml').data.decode(), '- title: foobar\n data_hash: KhGSreKJpp4AwDUWjtATeuAYLms=\n') self.assertEqual( writer.get( 'events/test_event/routes_data/KhGSreKJpp4AwDUWjtATeuAYLms='). data, b'\x81\xa6points\x90')
async def test_start_msgpack(self): repo = MemoryRepo() writer = TreeWriter(repo) writer.set_data('events/test_event/data.yaml', '{}'.encode()) writer.set_data('events/test_event/test_rider', b'\x91\x82\xa4time\xcbA\xd6\x1a\n\x98\x00\x00\x00\xa3bar\xa3foo') writer.commit('add test_event') app, settings = get_test_app_and_settings(repo) event = await Event.load(app, 'test_event', writer) tracker = await static_start_event_tracker(app, event, 'Test rider', {'name': 'test_rider', 'format': 'msgpack'}, None, None) await tracker.complete() self.assertEqual(len(tracker.points), 1) self.assertEqual(tracker.points[0], { 'time': datetime.datetime(2017, 1, 1), 'bar': 'foo', })
async def test_from_load(self): repo = MemoryRepo() writer = TreeWriter(repo) writer.set_data('events/test_event/data.yaml', ''' title: Test event '''.encode()) writer.commit('add test_event') app, settings = get_test_app_and_settings(repo) event = await Event.load(app, 'test_event', writer) self.assertEqual(event.config, {'title': 'Test event'}) self.assertEqual(event.routes, [])
async def test_load_events(self): repo = MemoryRepo() writer = TreeWriter(repo) writer.set_data('events/test_event/data.yaml', '{}'.encode()) writer.commit('add test_event') app, settings = get_test_app_and_settings(repo) await load_events(app, writer) events = app['trackers.events'] self.assertEqual(len(events), 1) event = events['test_event'] self.assertEqual(event.name, 'test_event')
def do_setup(self, data): repo = MemoryRepo() cache_dir = self.useFixture(fixtures.TempDir()) writer = TreeWriter(repo) writer.set_data('events/test_event/data.yaml', dedent(data).encode()) writer.commit('add test_event') async def start_mock_event_tracker(app, event, rider_name, tracker_data, start, end): tracker = Tracker('mock_tracker') tracker.completed.set_result(None) return tracker app, settings = get_test_app_and_settings(repo) settings['cache_path'] = cache_dir.path app['start_event_trackers'] = { 'mock': start_mock_event_tracker, } return app, settings, writer
async def add_gpx_to_event_routes(app, settings, args): import xml.etree.ElementTree as xml with open(args.gpx_file) as f: gpx_text = f.read() xml_doc = xml.fromstring(gpx_text) gpx_ns = { '1.0': { 'gpx': 'http://www.topografix.com/GPX/1/0', }, '1.1': { 'gpx': 'http://www.topografix.com/GPX/1/1', }, }[xml_doc.attrib['version']] trkpts = xml_doc.findall('./gpx:trk/gpx:trkseg/gpx:trkpt', gpx_ns) points = [[float(trkpt.attrib['lat']), float(trkpt.attrib['lon'])] for trkpt in trkpts] route = {'original_points': points} for key in ('no_elevation', 'split_at_dist', 'split_point_range', 'rdp_epsilon', 'circular_range', 'gpx_file'): route[key] = getattr(args, key) markers = [{ 'title': wpt.find('gpx:name', gpx_ns).text, 'svg_marker': { 'direction': 'sw', 'text': wpt.find('gpx:name', gpx_ns).text, }, 'position': { 'lat': round(float(wpt.attrib['lat']), 6), 'lng': round(float(wpt.attrib['lon']), 6), }, } for wpt in xml_doc.findall('./gpx:wpt', gpx_ns)] event_name = event_name_clean(args.event_name, settings) if not args.print: writer = TreeWriter(app['trackers.data_repo']) event = await trackers.events.Event.load(app, event_name, writer) await process_route(settings, route) if args.replace_main: event.routes[0] = route else: event.routes.append(route) process_secondary_route_details(event.routes) if 'markers' not in event.config: event.config['markers'] = [] event.config['markers'].extend(markers) # TODO - add gpx file to repo await event.save( f'{event_name}: add_gpx_to_event_routes {args.gpx_file}', tree_writer=writer, save_routes=True) else: original_points = route['original_points'] filtered_points = ( point for last_point, point in zip([None] + original_points[:-1], original_points) if point != last_point) point_points = route_with_distance_and_index(filtered_points) for point in point_points: print(f'{point.index}: {point.distance} {point.lat},{point.lng}')
async def test_from_load_with_routes(self): repo = MemoryRepo() writer = TreeWriter(repo) self.assertFalse(writer.exists('events/test_event/routes')) writer.set_data('events/test_event/data.yaml', '{}'.encode()) writer.set_data('events/test_event/routes.yaml', '- {data_hash: abcd, name: foo}'.encode()) writer.set_data('events/test_event/routes_data/abcd', b'\x81\xa6points\x90') writer.commit('add test_event') app, settings = get_test_app_and_settings(repo) event = await Event.load(app, 'test_event', writer) self.assertEqual(event.routes, [{'name': 'foo', 'points': []}])
async def test_implicit_static(self): data = ''' tracker_end: 2019-01-01 00:00:00 riders: - name: foo static_analyse: True ''' repo = MemoryRepo() cache_dir = self.useFixture(fixtures.TempDir()) writer = TreeWriter(repo) writer.set_data('events/test_event/data.yaml', dedent(data).encode()) writer.set_data('events/test_event/static/foo/source', msgpack.dumps([{ 'foo': 'bar' }], default=json_encode)) writer.set_data( 'events/test_event/static/foo/analyse', msgpack.dumps([{ 'foo': 'bar', 'speed': 1 }], default=json_encode)) writer.set_data('events/test_event/static/foo/off_route', msgpack.dumps([], default=json_encode)) writer.commit('add test_event') app, settings = get_test_app_and_settings(repo) settings['cache_path'] = cache_dir.path event = await Event.load(app, 'test_event', writer) await event.start_trackers() rider_objects = event.riders_objects['foo'] self.assertEqual(rider_objects.source_trackers[0].points, [{ 'foo': 'bar' }]) self.assertEqual(rider_objects.analyse_tracker.points, [{ 'foo': 'bar', 'speed': 1 }]) await event.stop_and_complete_trackers()
async def update_bounds(app, settings, args): event_name = event_name_clean(args.event_name, settings) writer = TreeWriter(app['trackers.data_repo']) event = await trackers.events.Event.load(app, event_name, writer) update_bounds_inner(event) await event.save(f'{event_name}: update_bounds', tree_writer=writer)
async def get_config(app, settings, args): tree_writer = TreeWriter(app['trackers.data_repo']) event_name = event_name_clean(args.event_name, settings) trackleaders_event_name = event_name try: event = await trackers.events.Event.load(app, event_name, tree_reader=tree_writer) except KeyError: event = await trackers.events.Event(app, event_name) logger.info('Downloading.') async with aiohttp.ClientSession() as session: race_page_text = await get( session, f'http://trackleaders.com/{trackleaders_event_name}f.php') riders_list_text = await get( session, f'http://trackleaders.com/spot/{trackleaders_event_name}/sortlist.php' ) route_text = await get( session, f'http://trackleaders.com/spot/{trackleaders_event_name}/route.js') markers_text = await get( session, f'http://trackleaders.com/spot/{trackleaders_event_name}/checkgen.js' ) logger.info('Scraping.') config = event.config race_page = bs4.BeautifulSoup(race_page_text, "html.parser") config['title'] = race_page.find('title').string.partition(' live')[0] # TODO Start time riders_list = bs4.BeautifulSoup(riders_list_text, "html.parser") riders_links = riders_list.find_all( 'a', title=re.compile('Open .* full <b>history</b>')) config['riders'] = [{ 'name': rider_link.string.strip(), 'name_short': rider_link.string.partition(' ')[0], 'tracker': { 'type': 'trackleaders', 'event': str(args.event_name), 'name': parse_qs(urlparse(rider_link['href']).query)['name'][0], }, } for rider_link in riders_links] route_js = es5(route_text) route_point_nodes = route_js.children()[0].elements[0].expr.right route_points = [[float(str(subnode)) for subnode in point_node] for point_node in route_point_nodes.items] event.routes = [ { 'original_points': route_points }, ] markers_js = es5(markers_text) markers_details = defaultdict(dict) for node in markers_js.children()[0].elements: if isinstance(node, ExprStatement): if isinstance(node.expr, Assign): id = node.expr.left.value if id.startswith('markercp'): markers_details[id]['location'] = [ float(str(loc_node)) for loc_node in node.expr.right.identifier.node.args.items[0].items ] if isinstance(node.expr, FunctionCall): id = node.expr.identifier.node.value if id.startswith( 'markercp' ) and node.expr.identifier.identifier.value == 'bindTooltip': markers_details[id]['text'] = node.expr.args.items[ 0].value[4:-5] config['markers'] = [{ 'marker_text': details['text'], 'position': { 'lat': details['location'][0], 'lng': details['location'][1], }, } for details in markers_details.values()] logger.info('Processing route.') await process_route(settings, event.routes[0]) process_secondary_route_details(event.routes) logger.info('Saving.') update_bounds_inner(event) assign_rider_colors_inner(event) await event.save(f'{event_name}: load config from trackleaders', tree_writer=tree_writer, save_routes=True)
def test(self): # TODO This test needs to be broken up. repo = MemoryRepo() writer = TreeWriter(repo) writer.set_data( 'a', b'file a', ) self.assertEqual(b'file a', writer.get('a').data) writer.commit('Add a.') self.assertEqual(b'file a', writer.get('a').data) self.assertEqual(b'file a', TreeReader(repo).get('a').data) writer.set_data( 'b/c', b'file c', ) writer.commit('Add b/c.') self.assertEqual(b'file c', writer.get('b/c').data) writer.set_data( 'b/c', b'file c ver 2', ) writer.commit('Modify b/c.') self.assertEqual(b'file c ver 2', writer.get('b/c').data) writer.remove('a') writer.commit('Remove a.') self.assertFalse(writer.exists('a')) writer.remove('b/c') writer.commit('Remove b/c.') self.assertFalse(writer.exists('b/c'))
async def analyse(app, settings, args): event_name = event_name_clean(args.event_name, settings) tree_writer = TreeWriter(app['trackers.data_repo']) event = await trackers.events.Event.load(app, event_name, tree_writer) await event.start_trackers(analyse=True) await event.stop_and_complete_trackers()