async def test_save_no_routes_before_and_after(self):
repo = MemoryRepo()
writer = TreeWriter(repo)
app, settings = get_test_app_and_settings(repo)
event = Event(app, 'test_event', {'title': 'Test event'}, [])
await event.save('save test_event',
tree_writer=writer,
save_routes=True)
self.assertFalse(writer.exists('events/test_event/routes'))
def test_rm_non_existant(self):
repo = MemoryRepo()
writer = TreeWriter(repo)
writer.set_data('a/b', b'b')
with self.assertRaises(KeyError):
writer.remove('a/c')
writer.set_data('a/d', b'd')
async def assign_rider_colors(app, settings, args):
event_name = event_name_clean(args.event_name, settings)
tree_writer = TreeWriter(app['trackers.data_repo'])
event = await trackers.events.Event.load(app, event_name, tree_writer)
assign_rider_colors_inner(event)
await event.save(f'{event_name}: assign_rider_colors',
tree_writer=tree_writer)
async def load_riders_from_csv(app, settings, args):
event_name = event_name_clean(args.event_name, settings)
tree_writer = TreeWriter(app['trackers.data_repo'])
event = await trackers.events.Event.load(app, event_name, tree_writer)
import csv
reader = csv.DictReader(sys.stdin)
def trackers_from_row(row):
if row.get('Traccar Device Id'):
yield {'type': 'traccar', 'unique_id': row['Traccar Device Id']}
if row.get('TKStorage'):
yield {'type': 'tkstorage', 'id': row['TKStorage']}
if row.get('Spot'):
yield {'type': 'spot', 'feed_id': row['Spot']}
event.config['riders'] = [{
'name': row['Name'],
'name_short': row['Short Name'],
'trackers': list(trackers_from_row(row)),
} for row in reader]
assign_rider_colors_inner(event)
await event.save(f'{event_name}: load_riders_from_csv',
tree_writer=tree_writer)
async def convert_to_static(app, settings, args):
tree_writer = TreeWriter(app['trackers.data_repo'])
event_name = event_name_clean(args.event_name, settings)
event = await trackers.events.Event.load(app, event_name, tree_writer)
await event.convert_to_static(tree_writer)
await event.store_analyse(tree_writer)
async def test_from_load_with_routes_old(self):
repo = MemoryRepo()
writer = TreeWriter(repo)
writer.set_data('events/test_event/data.yaml', '{}'.encode())
writer.set_data('events/test_event/routes', b'\x91\x81\xa6points\x90')
writer.commit('add test_event')
app, settings = get_test_app_and_settings(repo)
event = await Event.load(app, 'test_event', writer)
self.assertEqual(event.routes, [{'points': []}])
async def test_save_no_routes(self):
repo = MemoryRepo()
writer = TreeWriter(repo)
writer.set_data('events/test_event/data.yaml', '{}'.encode())
writer.set_data('events/test_event/routes', b'\x91\x81\xa6points\x90')
writer.commit('add test_event')
app, settings = get_test_app_and_settings(repo)
event = await Event.load(app, 'test_event', writer)
event.routes.pop()
await event.save('save test_event',
tree_writer=writer,
save_routes=True)
self.assertFalse(writer.exists('events/test_event/routes'))
async def test_start_json(self):
repo = MemoryRepo()
writer = TreeWriter(repo)
writer.set_data('events/test_event/data.yaml', '{}'.encode())
writer.set_data('events/test_event/test_rider', '[]'.encode())
writer.commit('add test_event')
app, settings = get_test_app_and_settings(repo)
event = await Event.load(app, 'test_event', writer)
tracker = await static_start_event_tracker(app, event, 'Test rider', {'name': 'test_rider', 'format': 'json'},
None, None)
await tracker.complete()
self.assertEqual(tracker.points, [])
async def reformat_event(app, settings, args):
event_name = event_name_clean(args.event_name, settings)
writer = TreeWriter(app['trackers.data_repo'])
event = await trackers.events.Event.load(app, event_name, writer)
for route in event.routes:
for key in ('start_distance', 'end_distance', 'dist_factor'):
if key in route:
route[key] = float(route[key])
await event.save(f'{event_name}: reformat_event',
tree_writer=writer,
save_routes=True)
async def process_event_routes(app, settings, args):
event_name = event_name_clean(args.event_name, settings)
writer = TreeWriter(app['trackers.data_repo'])
event = await trackers.events.Event.load(app, event_name, writer)
for route in event.routes:
if args.rdp_epsilon:
route['rdp_epsilon'] = args.rdp_epsilon
await process_route(settings, route)
process_secondary_route_details(event.routes)
await event.save(f'{event_name}: process_event_routes',
tree_writer=writer,
save_routes=True)
async def test_save(self):
repo = MemoryRepo()
writer = TreeWriter(repo)
app, settings = get_test_app_and_settings(repo)
event = Event(app, 'test_event', {'title': 'Test event'}, [{
'title': 'foobar',
'points': []
}])
await event.save('save test_event',
tree_writer=writer,
save_routes=True)
self.assertEqual(
writer.get('events/test_event/data.yaml').data.decode(),
'title: Test event\n')
self.assertEqual(
writer.get('events/test_event/routes.yaml').data.decode(),
'- title: foobar\n data_hash: KhGSreKJpp4AwDUWjtATeuAYLms=\n')
self.assertEqual(
writer.get(
'events/test_event/routes_data/KhGSreKJpp4AwDUWjtATeuAYLms=').
data, b'\x81\xa6points\x90')
async def test_start_msgpack(self):
repo = MemoryRepo()
writer = TreeWriter(repo)
writer.set_data('events/test_event/data.yaml', '{}'.encode())
writer.set_data('events/test_event/test_rider', b'\x91\x82\xa4time\xcbA\xd6\x1a\n\x98\x00\x00\x00\xa3bar\xa3foo')
writer.commit('add test_event')
app, settings = get_test_app_and_settings(repo)
event = await Event.load(app, 'test_event', writer)
tracker = await static_start_event_tracker(app, event, 'Test rider', {'name': 'test_rider', 'format': 'msgpack'},
None, None)
await tracker.complete()
self.assertEqual(len(tracker.points), 1)
self.assertEqual(tracker.points[0], {
'time': datetime.datetime(2017, 1, 1),
'bar': 'foo',
})
async def test_from_load(self):
repo = MemoryRepo()
writer = TreeWriter(repo)
writer.set_data('events/test_event/data.yaml', '''
title: Test event
'''.encode())
writer.commit('add test_event')
app, settings = get_test_app_and_settings(repo)
event = await Event.load(app, 'test_event', writer)
self.assertEqual(event.config, {'title': 'Test event'})
self.assertEqual(event.routes, [])
async def test_load_events(self):
repo = MemoryRepo()
writer = TreeWriter(repo)
writer.set_data('events/test_event/data.yaml', '{}'.encode())
writer.commit('add test_event')
app, settings = get_test_app_and_settings(repo)
await load_events(app, writer)
events = app['trackers.events']
self.assertEqual(len(events), 1)
event = events['test_event']
self.assertEqual(event.name, 'test_event')
def do_setup(self, data):
repo = MemoryRepo()
cache_dir = self.useFixture(fixtures.TempDir())
writer = TreeWriter(repo)
writer.set_data('events/test_event/data.yaml', dedent(data).encode())
writer.commit('add test_event')
async def start_mock_event_tracker(app, event, rider_name,
tracker_data, start, end):
tracker = Tracker('mock_tracker')
tracker.completed.set_result(None)
return tracker
app, settings = get_test_app_and_settings(repo)
settings['cache_path'] = cache_dir.path
app['start_event_trackers'] = {
'mock': start_mock_event_tracker,
}
return app, settings, writer
async def add_gpx_to_event_routes(app, settings, args):
import xml.etree.ElementTree as xml
with open(args.gpx_file) as f:
gpx_text = f.read()
xml_doc = xml.fromstring(gpx_text)
gpx_ns = {
'1.0': {
'gpx': 'http://www.topografix.com/GPX/1/0',
},
'1.1': {
'gpx': 'http://www.topografix.com/GPX/1/1',
},
}[xml_doc.attrib['version']]
trkpts = xml_doc.findall('./gpx:trk/gpx:trkseg/gpx:trkpt', gpx_ns)
points = [[float(trkpt.attrib['lat']),
float(trkpt.attrib['lon'])] for trkpt in trkpts]
route = {'original_points': points}
for key in ('no_elevation', 'split_at_dist', 'split_point_range',
'rdp_epsilon', 'circular_range', 'gpx_file'):
route[key] = getattr(args, key)
markers = [{
'title': wpt.find('gpx:name', gpx_ns).text,
'svg_marker': {
'direction': 'sw',
'text': wpt.find('gpx:name', gpx_ns).text,
},
'position': {
'lat': round(float(wpt.attrib['lat']), 6),
'lng': round(float(wpt.attrib['lon']), 6),
},
} for wpt in xml_doc.findall('./gpx:wpt', gpx_ns)]
event_name = event_name_clean(args.event_name, settings)
if not args.print:
writer = TreeWriter(app['trackers.data_repo'])
event = await trackers.events.Event.load(app, event_name, writer)
await process_route(settings, route)
if args.replace_main:
event.routes[0] = route
else:
event.routes.append(route)
process_secondary_route_details(event.routes)
if 'markers' not in event.config:
event.config['markers'] = []
event.config['markers'].extend(markers)
# TODO - add gpx file to repo
await event.save(
f'{event_name}: add_gpx_to_event_routes {args.gpx_file}',
tree_writer=writer,
save_routes=True)
else:
original_points = route['original_points']
filtered_points = (
point
for last_point, point in zip([None] +
original_points[:-1], original_points)
if point != last_point)
point_points = route_with_distance_and_index(filtered_points)
for point in point_points:
print(f'{point.index}: {point.distance} {point.lat},{point.lng}')
async def test_from_load_with_routes(self):
repo = MemoryRepo()
writer = TreeWriter(repo)
self.assertFalse(writer.exists('events/test_event/routes'))
writer.set_data('events/test_event/data.yaml', '{}'.encode())
writer.set_data('events/test_event/routes.yaml',
'- {data_hash: abcd, name: foo}'.encode())
writer.set_data('events/test_event/routes_data/abcd',
b'\x81\xa6points\x90')
writer.commit('add test_event')
app, settings = get_test_app_and_settings(repo)
event = await Event.load(app, 'test_event', writer)
self.assertEqual(event.routes, [{'name': 'foo', 'points': []}])
async def test_implicit_static(self):
data = '''
tracker_end: 2019-01-01 00:00:00
riders:
- name: foo
static_analyse: True
'''
repo = MemoryRepo()
cache_dir = self.useFixture(fixtures.TempDir())
writer = TreeWriter(repo)
writer.set_data('events/test_event/data.yaml', dedent(data).encode())
writer.set_data('events/test_event/static/foo/source',
msgpack.dumps([{
'foo': 'bar'
}], default=json_encode))
writer.set_data(
'events/test_event/static/foo/analyse',
msgpack.dumps([{
'foo': 'bar',
'speed': 1
}], default=json_encode))
writer.set_data('events/test_event/static/foo/off_route',
msgpack.dumps([], default=json_encode))
writer.commit('add test_event')
app, settings = get_test_app_and_settings(repo)
settings['cache_path'] = cache_dir.path
event = await Event.load(app, 'test_event', writer)
await event.start_trackers()
rider_objects = event.riders_objects['foo']
self.assertEqual(rider_objects.source_trackers[0].points, [{
'foo': 'bar'
}])
self.assertEqual(rider_objects.analyse_tracker.points, [{
'foo': 'bar',
'speed': 1
}])
await event.stop_and_complete_trackers()
async def update_bounds(app, settings, args):
event_name = event_name_clean(args.event_name, settings)
writer = TreeWriter(app['trackers.data_repo'])
event = await trackers.events.Event.load(app, event_name, writer)
update_bounds_inner(event)
await event.save(f'{event_name}: update_bounds', tree_writer=writer)
async def get_config(app, settings, args):
tree_writer = TreeWriter(app['trackers.data_repo'])
event_name = event_name_clean(args.event_name, settings)
trackleaders_event_name = event_name
try:
event = await trackers.events.Event.load(app,
event_name,
tree_reader=tree_writer)
except KeyError:
event = await trackers.events.Event(app, event_name)
logger.info('Downloading.')
async with aiohttp.ClientSession() as session:
race_page_text = await get(
session, f'http://trackleaders.com/{trackleaders_event_name}f.php')
riders_list_text = await get(
session,
f'http://trackleaders.com/spot/{trackleaders_event_name}/sortlist.php'
)
route_text = await get(
session,
f'http://trackleaders.com/spot/{trackleaders_event_name}/route.js')
markers_text = await get(
session,
f'http://trackleaders.com/spot/{trackleaders_event_name}/checkgen.js'
)
logger.info('Scraping.')
config = event.config
race_page = bs4.BeautifulSoup(race_page_text, "html.parser")
config['title'] = race_page.find('title').string.partition(' live')[0]
# TODO Start time
riders_list = bs4.BeautifulSoup(riders_list_text, "html.parser")
riders_links = riders_list.find_all(
'a', title=re.compile('Open .* full <b>history</b>'))
config['riders'] = [{
'name': rider_link.string.strip(),
'name_short': rider_link.string.partition(' ')[0],
'tracker': {
'type': 'trackleaders',
'event': str(args.event_name),
'name': parse_qs(urlparse(rider_link['href']).query)['name'][0],
},
} for rider_link in riders_links]
route_js = es5(route_text)
route_point_nodes = route_js.children()[0].elements[0].expr.right
route_points = [[float(str(subnode)) for subnode in point_node]
for point_node in route_point_nodes.items]
event.routes = [
{
'original_points': route_points
},
]
markers_js = es5(markers_text)
markers_details = defaultdict(dict)
for node in markers_js.children()[0].elements:
if isinstance(node, ExprStatement):
if isinstance(node.expr, Assign):
id = node.expr.left.value
if id.startswith('markercp'):
markers_details[id]['location'] = [
float(str(loc_node)) for loc_node in
node.expr.right.identifier.node.args.items[0].items
]
if isinstance(node.expr, FunctionCall):
id = node.expr.identifier.node.value
if id.startswith(
'markercp'
) and node.expr.identifier.identifier.value == 'bindTooltip':
markers_details[id]['text'] = node.expr.args.items[
0].value[4:-5]
config['markers'] = [{
'marker_text': details['text'],
'position': {
'lat': details['location'][0],
'lng': details['location'][1],
},
} for details in markers_details.values()]
logger.info('Processing route.')
await process_route(settings, event.routes[0])
process_secondary_route_details(event.routes)
logger.info('Saving.')
update_bounds_inner(event)
assign_rider_colors_inner(event)
await event.save(f'{event_name}: load config from trackleaders',
tree_writer=tree_writer,
save_routes=True)
def test(self):
# TODO This test needs to be broken up.
repo = MemoryRepo()
writer = TreeWriter(repo)
writer.set_data(
'a',
b'file a',
)
self.assertEqual(b'file a', writer.get('a').data)
writer.commit('Add a.')
self.assertEqual(b'file a', writer.get('a').data)
self.assertEqual(b'file a', TreeReader(repo).get('a').data)
writer.set_data(
'b/c',
b'file c',
)
writer.commit('Add b/c.')
self.assertEqual(b'file c', writer.get('b/c').data)
writer.set_data(
'b/c',
b'file c ver 2',
)
writer.commit('Modify b/c.')
self.assertEqual(b'file c ver 2', writer.get('b/c').data)
writer.remove('a')
writer.commit('Remove a.')
self.assertFalse(writer.exists('a'))
writer.remove('b/c')
writer.commit('Remove b/c.')
self.assertFalse(writer.exists('b/c'))
async def analyse(app, settings, args):
event_name = event_name_clean(args.event_name, settings)
tree_writer = TreeWriter(app['trackers.data_repo'])
event = await trackers.events.Event.load(app, event_name, tree_writer)
await event.start_trackers(analyse=True)
await event.stop_and_complete_trackers()