def process(input_paths, output_url, output_format, tags, output_batch_size,
start_date_datum, start_time_datum):
data = DataState()
time_util = TimeUtils(start_date_datum, start_time_datum)
data.set_data_item('DateTime',
time_util.calculateTimestamp(None, None, 0.0))
last_data_time = data.state_time
parser = Parser(data)
formatter = FormatterFactory.getFormatter(output_format, tags)
ostream = io.StringIO()
header_line = formatter.formatHeading(data)
if header_line is not None:
if verbose > 0:
print(header_line)
print(header_line, end='\n', file=ostream)
line_count = 0
for path in input_paths:
print("Processing {0}".format(path))
with open(path, "r") as ifile:
for data_line in ifile:
input_line = data_line.strip()
if verbose > 0:
print(input_line)
parser.parse(input_line)
output_lines = formatter.formatData(data)
if output_lines is not None:
for output_line in output_lines:
if verbose > 0:
print(output_line)
print(output_line, end='\n', file=ostream)
line_count = line_count + 1
if data.state_time > last_data_time:
last_data_time = data.state_time
data.set_data_item(
'DateTime',
time_util.calculateTimestamp(
data.get_data_item('GPS_Date'),
data.get_data_item('GPS_Time'), data.state_time))
output_line = formatter.formatTimeIncrement(data)
if output_line is not None:
if verbose > 0:
print(output_line)
print(output_line, end='\n', file=ostream)
line_count = line_count + 1
if line_count >= output_batch_size:
writeBatch(output_url, ostream)
ostream.close()
ostream = io.StringIO()
line_count = 0
footer_line = formatter.formatFooter(data)
if footer_line is not None:
if verbose > 0:
print(footer_line)
print(footer_line, end='\n', file=ostream)
writeBatch(output_url, ostream)
ostream.close()
class Test(unittest.TestCase):
def setUp(self):
self.test_state = DataState()
self.test_state.set_data_item('Time', 321.36)
self.test_state.set_data_item('DateTime', datetime(2019,5,15,21,23,48,21000,timezone.utc))
self.test_state.set_data_item('GPS_Time', '21:23:48.021')
self.test_state.set_data_item('GPS_Date', '15-05-19')
self.test_state.set_data_item('Latitude', 39.739242)
self.test_state.set_data_item('Longitude', 104.9903299)
self.test_state.set_data_item('Altitude', 1609)
self.test_state.set_data_item('X', 0.15)
self.test_state.set_data_item('Y', 0.86)
self.test_state.set_data_item('Z', 1.18)
self.test_state.set_data_item('KPH', 86.3)
self.test_state.set_data_item('Lap', 3)
self.test_state.set_data_item('RPM', 5891)
self.test_state.set_data_item('Gear', 3)
self.test_state.set_data_item('Throttle', 97)
self.test_state.set_data_item('ECU_Throttle', 95)
self.test_state.set_data_item('Brake', 1)
self.test_state.set_data_item('Brake_Pressure', 42)
self.test_state.set_data_item('Clutch', 0)
self.test_state.set_data_item('Steering_Angle', 6.21)
self.test_state.set_data_item('Steering_Rate', 0.42)
self.test_state.set_data_item('Coolant_Temperature', 99)
self.test_state.set_data_item('Oil_Pressure', 240)
self.test_state.set_data_item('Oil_Temperature', 96)
self.test_state.set_data_item('LF_KPH', 85.2)
self.test_state.set_data_item('RF_KPH', 86.9)
self.test_state.set_data_item('LR_KPH', 85.0)
self.test_state.set_data_item('RR_KPH', 86.7)
self.test_state.set_data_item('PSM', 0)
self.test_state.set_data_item('GPS_KPH', 0.86)
self.test_state.set_data_item('GPS_Heading', 321.41)
self.test_state.set_data_item('Accuracy', 0.10)
self.test_formatter = ThingSpeakFormatter(dict())
pass
def testFormatDate(self):
actual_json = self.test_formatter.formatTimeIncrement(self.test_state)
print(actual_json)
actual = json.loads(actual_json)
self.assertAlmostEqual(39.739242, actual['Latitude'], 6)
self.assertAlmostEqual(104.9903299, actual['Longitude'], 6)
self.assertAlmostEqual(1609, actual['Elevation'], 3)
self.assertAlmostEqual(0.15, actual['Field1'], 3)
self.assertAlmostEqual(0.86, actual['Field2'], 3)
self.assertAlmostEqual(1.18, actual['Field3'], 3)
class CsvFormatterTest(unittest.TestCase):
def setUp(self):
self.test_state = DataState()
self.test_state.set_data_item('Time', 321.36)
self.test_state.set_data_item('GPS_Time', '21:23:48.021')
self.test_state.set_data_item('GPS_Date', '15-05-19')
self.test_state.set_data_item('Latitude', 39.739242)
self.test_state.set_data_item('Longitude', 104.9903299)
self.test_state.set_data_item('Altitude', 1609)
self.test_state.set_data_item('X', 0.15)
self.test_state.set_data_item('Y', 0.86)
self.test_state.set_data_item('Z', 1.18)
self.test_state.set_data_item('KPH', 86.3)
self.test_state.set_data_item('Lap', 3)
self.test_state.set_data_item('RPM', 5891)
self.test_state.set_data_item('Gear', 3)
self.test_state.set_data_item('Throttle', 97)
self.test_state.set_data_item('ECU_Throttle', 95)
self.test_state.set_data_item('Brake', 1)
self.test_state.set_data_item('Brake_Pressure', 42)
self.test_state.set_data_item('Clutch', 0)
self.test_state.set_data_item('Steering_Angle', 6.21)
self.test_state.set_data_item('Steering_Rate', 0.42)
self.test_state.set_data_item('Coolant_Temperature', 99)
self.test_state.set_data_item('Oil_Pressure', 240)
self.test_state.set_data_item('Oil_Temperature', 96)
self.test_state.set_data_item('LF_KPH', 85.2)
self.test_state.set_data_item('RF_KPH', 86.9)
self.test_state.set_data_item('LR_KPH', 85.0)
self.test_state.set_data_item('RR_KPH', 86.7)
self.test_state.set_data_item('PSM', 0)
self.test_state.set_data_item('GPS_KPH', 0.86)
self.test_state.set_data_item('GPS_Heading', 321.41)
self.test_state.set_data_item('Accuracy', 0.10)
self.test_formatter = CsvFormatter(dict())
pass
def testFormatHeader(self):
test_csv_line = self.test_formatter.formatHeading(self.test_state)
self.assertEqual("Time, GPS_Time, GPS_Date, Latitude, Longitude, Altitude, X, Y, Z, KPH, Lap, RPM, Gear, Throttle, ECU_Throttle, Brake, Brake_Pressure, Clutch, Steering_Angle, Steering_Rate, Coolant_Temperature, Oil_Pressure, Oil_Temperature, LF_KPH, RF_KPH, LR_KPH, RR_KPH, PSM, GPS_KPH, GPS_Heading, Accuracy, Sport_Mode, Pasm_Sport_Mode, PSM_Disable",
test_csv_line,
"The generated CSV line doesn't match the expected value.")
pass
def testFormatLine(self):
test_csv_line = self.test_formatter.formatTimeIncrement(self.test_state)
self.assertEqual("321.36, 21:23:48.021, 15-05-19, 39.7392, 104.9903, 1609, 0.15, 0.86, 1.18, 86.30, 3, 5891, 3, 97, 95, 1, 42, 0, 6.21, 0.42, 99, 240, 96, 85.20, 86.90, 85.00, 86.70, 0, 0.86, 321.41, 0.10, , , ",
test_csv_line,
"The generated CSV line doesn't match the expected value.")
pass
class InfluxDbFormatterTest(unittest.TestCase):
def setUp(self):
self.test_formatter = InfluxDbFormatter(dict(vin='WP0AB29858U782772', vehicle='cayman', mileage='79832', temperature='24'))
pass
def testLocation(self):
self.test_state = DataState()
self.test_state.set_data_item('Time', 321.36)
self.test_state.set_data_item('DateTime', datetime(2019,5,15,21,23,48,21000,timezone.utc))
self.test_state.set_data_item('GPS_Time', '21:23:48.021')
self.test_state.set_data_item('GPS_Date', '15-05-19')
self.test_state.set_data_item('Latitude', 39.739242)
self.test_state.set_data_item('Longitude', 104.9903299)
self.test_state.set_data_item('Altitude', 1609)
self.test_state.set_data_item('Lap', 3)
actual = self.test_formatter.formatData(self.test_state)
expected = 'location,vehicle=cayman,vin=WP0AB29858U782772,mileage=79832,lap=3,geohash=00000000 latitude=39.739242,longitude=104.9903299,altitude=1609 1557955428021'
self.assertEqual(expected, actual[0], 'The generated influxdb line protocol data for locaation was not as expected.')
self.assertTrue('Latitude' not in self.test_state.get_dirty_fields(), 'Latitude still dirty after formatting.')
self.assertTrue('Longitude' not in self.test_state.get_dirty_fields(), 'Longitude still dirty after formatting.')
#self.assertTrue('Altitude' not in self.test_state.get_dirty_fields(), 'Altitude still dirty after formatting.')
def testAcceleration(self):
self.test_state = DataState()
self.test_state.set_data_item('Time', 321.36)
self.test_state.set_data_item('DateTime', datetime(2019,5,15,21,23,48,21000,timezone.utc))
self.test_state.set_data_item('GPS_Time', '21:23:48.021')
self.test_state.set_data_item('GPS_Date', '15-05-19')
self.test_state.set_data_item('X', 0.15)
self.test_state.set_data_item('Y', 0.86)
self.test_state.set_data_item('Z', 1.18)
self.test_state.set_data_item('Lap', 3)
actual = self.test_formatter.formatData(self.test_state)
expected = 'acceleration,vehicle=cayman,vin=WP0AB29858U782772,mileage=79832,lap=3 x=0.15,y=0.86,z=1.18 1557955428021'
self.assertEqual(expected, actual[0], 'The generated influxdb line protocol data for acceleration was not as expected.')
self.assertTrue('X' not in self.test_state.get_dirty_fields(), 'X still dirty after formatting.')
self.assertTrue('Y' not in self.test_state.get_dirty_fields(), 'Y still dirty after formatting.')
self.assertTrue('Z' not in self.test_state.get_dirty_fields(), 'Z still dirty after formatting.')
def testSteering(self):
self.test_state = DataState()
self.test_state.set_data_item('Time', 321.36)
self.test_state.set_data_item('DateTime', datetime(2019,5,15,21,23,48,21000,timezone.utc))
self.test_state.set_data_item('Steering_Angle', 6.21)
self.test_state.set_data_item('Steering_Rate', 0.42)
self.test_state.set_data_item('Lap', 3)
actual = self.test_formatter.formatData(self.test_state)
expected = 'steering,vehicle=cayman,vin=WP0AB29858U782772,mileage=79832,lap=3 angle=6.21,rate=0.42 1557955428021'
self.assertEqual(expected, actual[0], 'The generated influxdb line protocol data for steering was not as expected.')
self.assertTrue('Steering_Angle' not in self.test_state.get_dirty_fields(), 'Steering_Angle still dirty after formatting.')
self.assertTrue('Steering_Rate' not in self.test_state.get_dirty_fields(), 'Steering_Rate still dirty after formatting.')
def testThrottle(self):
self.test_state = DataState()
self.test_state.set_data_item('Time', 321.36)
self.test_state.set_data_item('DateTime', datetime(2019,5,15,21,23,48,21000,timezone.utc))
self.test_state.set_data_item('Throttle', 97)
self.test_state.set_data_item('ECU_Throttle', 95)
self.test_state.set_data_item('Lap', 3)
actual = self.test_formatter.formatData(self.test_state)
expected = 'throttle,vehicle=cayman,vin=WP0AB29858U782772,mileage=79832,lap=3 driver=97,ecu=95 1557955428021'
self.assertEqual(expected, actual[0], 'The generated influxdb line protocol data for throttle was not as expected.')
self.assertTrue('Throttle' not in self.test_state.get_dirty_fields(), 'Throttle still dirty after formatting.')
self.assertTrue('ECU_Throttle' not in self.test_state.get_dirty_fields(), 'ECU_Throttle still dirty after formatting.')
def testBrake(self):
self.test_state = DataState()
self.test_state.set_data_item('Time', 321.36)
self.test_state.set_data_item('DateTime', datetime(2019,5,15,21,23,48,21000,timezone.utc))
self.test_state.set_data_item('Brake', 1)
self.test_state.set_data_item('Brake_Pressure', 42)
self.test_state.set_data_item('Lap', 3)
actual = self.test_formatter.formatData(self.test_state)
expected = 'brake,vehicle=cayman,vin=WP0AB29858U782772,mileage=79832,lap=3 applied=1,pressure=42 1557955428021'
self.assertEqual(expected, actual[0], 'The generated influxdb line protocol data for brake was not as expected.')
self.assertTrue('Brake' not in self.test_state.get_dirty_fields(), 'Brake still dirty after formatting.')
self.assertTrue('Brake_Pressure' not in self.test_state.get_dirty_fields(), 'Brake_Pressure still dirty after formatting.')
def testClutch(self):
self.test_state = DataState()
self.test_state.set_data_item('Time', 321.36)
self.test_state.set_data_item('DateTime', datetime(2019,5,15,21,23,48,21000,timezone.utc))
self.test_state.set_data_item('Clutch', 0)
self.test_state.set_data_item('Lap', 3)
actual = self.test_formatter.formatData(self.test_state)
expected = 'clutch,vehicle=cayman,vin=WP0AB29858U782772,mileage=79832,lap=3 released=0 1557955428021'
self.assertEqual(expected, actual[0], 'The generated influxdb line protocol data for clutch was not as expected.')
self.assertTrue('Clutch' not in self.test_state.get_dirty_fields(), 'Clutch still dirty after formatting.')
def testRPM(self):
self.test_state = DataState()
self.test_state.set_data_item('Time', 321.36)
self.test_state.set_data_item('DateTime', datetime(2019,5,15,21,23,48,21000,timezone.utc))
self.test_state.set_data_item('RPM', 5996)
self.test_state.set_data_item('Lap', 3)
actual = self.test_formatter.formatData(self.test_state)
expected = 'rpm,vehicle=cayman,vin=WP0AB29858U782772,mileage=79832,lap=3 value=5996 1557955428021'
self.assertEqual(expected, actual[0], 'The generated influxdb line protocol data for rpm was not as expected.')
self.assertTrue('RPM' not in self.test_state.get_dirty_fields(), 'RPM still dirty after formatting.')
def testCoolantTemperature(self):
self.test_state = DataState()
self.test_state.set_data_item('Time', 321.36)
self.test_state.set_data_item('DateTime', datetime(2019,5,15,21,23,48,21000,timezone.utc))
self.test_state.set_data_item('Coolant_Temperature', 99)
self.test_state.set_data_item('Lap', 3)
actual = self.test_formatter.formatData(self.test_state)
expected = 'coolant_temperature,vehicle=cayman,vin=WP0AB29858U782772,mileage=79832,lap=3 value=99 1557955428021'
self.assertEqual(expected, actual[0], 'The generated influxdb line protocol data for coolant_temperature was not as expected.')
self.assertTrue('Coolant_Temperature' not in self.test_state.get_dirty_fields(), 'Coolant_Temperature still dirty after formatting.')
def testOilPressure(self):
self.test_state = DataState()
self.test_state.set_data_item('Time', 321.36)
self.test_state.set_data_item('DateTime', datetime(2019,5,15,21,23,48,21000,timezone.utc))
self.test_state.set_data_item('Oil_Pressure', 240)
self.test_state.set_data_item('Lap', 3)
actual = self.test_formatter.formatData(self.test_state)
expected = 'oil_pressure,vehicle=cayman,vin=WP0AB29858U782772,mileage=79832,lap=3 value=240 1557955428021'
self.assertEqual(expected, actual[0], 'The generated influxdb line protocol data for oil_pressure was not as expected.')
self.assertTrue('Oil_Pressure' not in self.test_state.get_dirty_fields(), 'Oil_Pressure still dirty after formatting.')
def testOilTemperature(self):
self.test_state = DataState()
self.test_state.set_data_item('Time', 321.36)
self.test_state.set_data_item('DateTime', datetime(2019,5,15,21,23,48,21000,timezone.utc))
self.test_state.set_data_item('Oil_Temperature', 96)
self.test_state.set_data_item('Lap', 3)
actual = self.test_formatter.formatData(self.test_state)
expected = 'oil_temperature,vehicle=cayman,vin=WP0AB29858U782772,mileage=79832,lap=3 value=96 1557955428021'
self.assertEqual(expected, actual[0], 'The generated influxdb line protocol data for oil_temperature was not as expected.')
self.assertTrue('Oil_Temperature' not in self.test_state.get_dirty_fields(), 'Oil_Temperature still dirty after formatting.')
def testWheelSpeed(self):
self.test_state = DataState()
self.test_state.set_data_item('Time', 321.36)
self.test_state.set_data_item('DateTime', datetime(2019,5,15,21,23,48,21000,timezone.utc))
self.test_state.set_data_item('LF_KPH', 85.2)
self.test_state.set_data_item('RF_KPH', 86.9)
self.test_state.set_data_item('LR_KPH', 85.0)
self.test_state.set_data_item('RR_KPH', 86.7)
self.test_state.set_data_item('Lap', 3)
actual = self.test_formatter.formatData(self.test_state)
expected = 'wheel_speed,vehicle=cayman,vin=WP0AB29858U782772,mileage=79832,lap=3 left_front=85.2,right_front=86.9,left_rear=85.0,right_rear=86.7 1557955428021'
self.assertEqual(expected, actual[0], 'The generated influxdb line protocol data for wheel_speed was not as expected.')
self.assertTrue('LF_KPH' not in self.test_state.get_dirty_fields(), 'LF_KPH still dirty after formatting.')
self.assertTrue('RF_KPH' not in self.test_state.get_dirty_fields(), 'RF_KPH still dirty after formatting.')
self.assertTrue('LR_KPH' not in self.test_state.get_dirty_fields(), 'LR_KPH still dirty after formatting.')
self.assertTrue('RR_KPH' not in self.test_state.get_dirty_fields(), 'RR_KPH still dirty after formatting.')
def testPModes(self):
self.test_state = DataState()
self.test_state.set_data_item('Time', 321.36)
self.test_state.set_data_item('DateTime', datetime(2019,5,15,21,23,48,21000,timezone.utc))
self.test_state.set_data_item('Sport_Mode', 1)
self.test_state.set_data_item('Pasm_Sport_Mode', 1)
self.test_state.set_data_item('PSM_Disable', 0)
self.test_state.set_data_item('Lap', 3)
actual = self.test_formatter.formatData(self.test_state)
expected = 'pmodes,vehicle=cayman,vin=WP0AB29858U782772,mileage=79832,lap=3 sport=1,pasm_sport=1,psm_off=0 1557955428021'
self.assertEqual(expected, actual[0], 'The generated influxdb line protocol data for pmodes was not as expected.')
self.assertTrue('Sport_Mode' not in self.test_state.get_dirty_fields(), 'Sport_Mode still dirty after formatting.')
self.assertTrue('Pasm_Sport_Mode' not in self.test_state.get_dirty_fields(), 'Pasm_Sport_Mode still dirty after formatting.')
self.assertTrue('PSM_Disable' not in self.test_state.get_dirty_fields(), 'PSM_Disable still dirty after formatting.')