def test_default(self):
lines = LINES.copy()
expected_results = RESULTS.copy()
# Use port wind speed, output in m/s
tw = TrueWindsTransform(data_id='truw',
course_fields='S330CourseTrue',
speed_fields='S330Speed',
heading_fields='S330HeadingTrue',
wind_dir_fields='MwxPortRelWindDir',
wind_speed_fields='MwxPortRelWindSpeed',
convert_speed_factor=0.5144)
parse = ParseNMEATransform()
while lines:
record = parse.transform(lines.pop(0))
result = tw.transform(record)
expected = expected_results.pop(0)
if expected:
self.assertEqual(result.data_id, 'truw')
self.assertEqual(result.message_type, None)
self.assertEqual(result.timestamp, expected['timestamp'])
self.assertAlmostEqual(result.fields['ApparentWindDir'],
expected['fields']['ApparentWindDir'],
delta=0.00001)
self.assertAlmostEqual(result.fields['TrueWindDir'],
expected['fields']['TrueWindDir'],
delta=0.00001)
self.assertAlmostEqual(result.fields['TrueWindSpeed'],
expected['fields']['TrueWindSpeed'],
delta=0.00001)
self.assertDictEqual(result.metadata, expected['metadata'])
else:
self.assertIsNone(result)
return
def test_error(self):
p = ParseNMEATransform()
q = QCFilterTransform(bounds='KnudLFDepth:0:6000,KnudHFDepth:0:5000')
record = 'knud 2017-11-04:05:12:21.981359'
self.assertEqual(q.transform(record),
'Improper format record: knud 2017-11-04:05:12:21.981359')
record = p.transform('knud 2017-11-04:05:12:21.981359 3.5kHz,5146.29,0,,,,1500,-39.583558,-37.466183')
self.assertEqual(q.transform(record),
'KnudHFDepth: non-numeric value: "None"')
def test_json(self):
transform = ParseNMEATransform(json=True)
self.assertIsNone(transform.transform(None))
for i in range(len(LINES)):
line = LINES[i]
record = RECORDS[i]
logging.debug('line: %s', line)
result = transform.transform(line)
logging.debug('result: %s', result)
self.assertEqual(ordered(json.loads(result)), ordered(json.loads(JSON[i])))
def test_error(self):
p = ParseNMEATransform()
q = QCFilterTransform(bounds='KnudLFDepth:0:6000,KnudHFDepth:0:5000')
record = 'knud 2017-11-04T05:12:21.981359Z'
self.assertEqual(
q.transform(record),
'Record passed to QCFilterTransform was neither a dict nor a DASRecord. Type was <class \'str\'>: knud 2017-11-04T05:12:21.981359Z'
)
record = p.transform(
'knud 2017-11-04T05:12:21.981359Z 3.5kHz,5146.29,0,,,,1500,-39.583558,-37.466183'
)
self.assertEqual(q.transform(record),
'KnudHFDepth: non-numeric value: "None"')
def test_message(self):
p = ParseNMEATransform()
q = QCFilterTransform(bounds='KnudLFDepth:0:6000,KnudHFDepth:0:5000',
message='The sky is falling!')
record = 'knud 2017-11-04:05:12:21.981359'
self.assertEqual(q.transform(record), 'The sky is falling!')
def test_default(self):
transform = ParseNMEATransform()
self.assertIsNone(transform.transform(None))
for i in range(len(LINES)):
line = LINES[i]
record = RECORDS[i]
logging.debug('line: %s', line)
result = transform.transform(line)
logging.debug('result: %s', result)
self.assertEqual(record['data_id'], result.data_id)
self.assertEqual(record['message_type'], result.message_type)
self.assertEqual(record['timestamp'], result.timestamp)
self.assertDictEqual(record['fields'], result.fields)
self.assertDictEqual(record['metadata'], result.metadata)
def test_field_dict(self):
parse = ParseNMEATransform()
recip = RecipTransform()
port_winds = TrueWindsTransform(
course_field='S330CourseTrue',
speed_field='S330Speed',
heading_field='S330HeadingTrue',
wind_dir_field='MwxPortRelWindDir',
wind_speed_field='MwxPortRelWindSpeed',
true_dir_name='PortTrueWindDir',
true_speed_name='PortTrueWindSpeed',
apparent_dir_name='PortApparentWindDir',
convert_speed_factor=0.5144)
stbd_winds = TrueWindsTransform(
course_field='S330CourseTrue',
speed_field='S330Speed',
heading_field='S330HeadingTrue',
wind_dir_field='MwxStbdRelWindDir',
wind_speed_field='MwxStbdRelWindSpeed',
true_dir_name='StbdTrueWindDir',
true_speed_name='StbdTrueWindSpeed',
apparent_dir_name='StbdApparentWindDir',
convert_speed_factor=0.5144)
# Test using DASRecords as input
t = ComposedDerivedDataTransform(
transforms=[recip, port_winds, stbd_winds])
field_values = {}
# Build the field dict
for i in range(len(LINES)):
line = LINES[i]
record = parse.transform(line)
if not record:
continue
for field, value in record.fields.items():
if not field in field_values:
field_values[field] = []
field_values[field].append([record.timestamp, value])
results = t.transform(field_values)
self.assertRecursiveAlmostEqual(results, FIELD_DICT_RESULT)
"""
def test_message(self):
p = ParseNMEATransform()
q = QCFilterTransform(bounds='KnudLFDepth:0:6000,KnudHFDepth:0:5000',
message='The sky is falling!')
record = {'KnudLFDepth': 5999}
self.assertEqual(q.transform(record), None)
record = {'KnudLFDepth': 6001}
self.assertEqual(q.transform(record), 'The sky is falling!')
def test_das_record(self):
"""Test using DASRecords as input."""
parse = ParseNMEATransform()
recip = RecipTransform()
port_winds = TrueWindsTransform(
course_field='S330CourseTrue',
speed_field='S330Speed',
heading_field='S330HeadingTrue',
wind_dir_field='MwxPortRelWindDir',
wind_speed_field='MwxPortRelWindSpeed',
true_dir_name='PortTrueWindDir',
true_speed_name='PortTrueWindSpeed',
apparent_dir_name='PortApparentWindDir',
convert_speed_factor=0.5144)
stbd_winds = TrueWindsTransform(
course_field='S330CourseTrue',
speed_field='S330Speed',
heading_field='S330HeadingTrue',
wind_dir_field='MwxStbdRelWindDir',
wind_speed_field='MwxStbdRelWindSpeed',
true_dir_name='StbdTrueWindDir',
true_speed_name='StbdTrueWindSpeed',
apparent_dir_name='StbdApparentWindDir',
convert_speed_factor=0.5144)
t = ComposedDerivedDataTransform(
transforms=[recip, port_winds, stbd_winds])
for i in range(len(LINES)):
line = LINES[i]
record = parse.transform(line)
result = t.transform(record)
expected = DAS_RECORD_RESULTS[i]
logging.info('Input fields: %s', record.fields)
logging.info('Got result: %s', result)
logging.info('Expected result: %s', expected)
if not result or not expected:
self.assertIsNone(result)
self.assertIsNone(expected)
else:
self.assertRecursiveAlmostEqual(result.fields, expected)
def test_nmea(self):
lines = LINES.copy()
expected_results = RESULTS.copy()
nmea_results = NMEA_RESULTS.copy()
# Use port wind speed, output in m/s
tw = TrueWindsTransform(data_id='truw',
course_fields='S330CourseTrue',
speed_fields='S330Speed',
heading_fields='S330HeadingTrue',
wind_dir_fields='MwxPortRelWindDir',
wind_speed_fields='MwxPortRelWindSpeed',
convert_speed_factor=0.5144,
output_nmea=True)
parse = ParseNMEATransform()
while lines:
record = parse.transform(lines.pop(0))
result = tw.transform(record)
nmea_expected = nmea_results.pop(0)
self.assertEqual(result, nmea_expected)
# Now check that parsing into a DASRecord works as expected
das_result = parse.transform(result)
expected = expected_results.pop(0)
if das_result is None:
self.assertEqual(das_result, expected)
else:
das_fields = das_result.fields
exp_fields = expected['fields']
self.assertAlmostEqual(das_fields['TrueWindDir'],
exp_fields['TrueWindDir'],
delta=0.001)
self.assertAlmostEqual(das_fields['TrueWindSpeed'],
exp_fields['TrueWindSpeed'],
delta=0.001)
self.assertAlmostEqual(das_fields['ApparentWindDir'],
exp_fields['ApparentWindDir'],
delta=0.001)
return
def test_default(self):
lines = LINES.copy()
expected_results = RESULTS.copy()
# Use port wind speed, output in m/s
tw = TrueWindsTransform(course_field='S330CourseTrue',
speed_field='S330Speed',
heading_field='S330HeadingTrue',
wind_dir_field='MwxPortRelWindDir',
update_on_fields=['MwxPortRelWindDir'],
wind_speed_field='MwxPortRelWindSpeed',
true_dir_name='PortTrueWindDir',
true_speed_name='PortTrueWindSpeed',
apparent_dir_name='PortApparentWindDir',
convert_speed_factor=0.5144)
parse = ParseNMEATransform()
value_dict = {}
timestamp_dict = {}
while lines:
record = parse.transform(lines.pop(0))
result_list = tw.transform(record)
self.assertEqual(type(result_list), list)
result = result_list[0] if len(result_list) else None
expected = expected_results.pop(0)
logging.debug('Got result: %s', result)
logging.debug('Expected result: %s\n', expected)
if not result or not expected:
self.assertIsNone(result)
self.assertIsNone(expected)
else:
logging.debug('Comparing result:\n%s\nwith expected:\n%s',
result.fields, expected)
self.assertRecursiveAlmostEqual(result.fields, expected)
def test_no_timestamps(self):
lines = LINES.copy()
expected_results = NO_TIMESTAMP_RESULTS.copy()
# Use port wind speed, output in m/s
tw = TrueWindsTransform(course_field='S330CourseTrue',
speed_field='S330Speed',
heading_field='S330HeadingTrue',
wind_dir_field='MwxPortRelWindDir',
update_on_fields=['MwxPortRelWindDir'],
wind_speed_field='MwxPortRelWindSpeed',
true_dir_name='PortTrueWindDir',
true_speed_name='PortTrueWindSpeed',
apparent_dir_name='PortApparentWindDir',
convert_speed_factor=0.5144)
parse = ParseNMEATransform()
value_dict = {}
timestamp_dict = {}
while lines:
record = parse.transform(lines.pop(0))
for field in record.fields:
value_dict[field] = record.fields[field]
timestamp_dict[field] = record.timestamp
result = tw.transform(value_dict)
expected = expected_results.pop(0)
logging.info('Got result: %s', result)
logging.info('Expected result: %s', expected)
if not result or not expected:
self.assertIsNone(result)
self.assertIsNone(expected)
else:
self.assertRecursiveAlmostEqual(result, expected)
def test_check_format(self):
f1_name = self.tmpdirname + '/f1'
f2_name = self.tmpdirname + '/f2'
# This should be okay
ComposedWriter(
transforms=[PrefixTransform('prefix')],
writers=[TextFileWriter(f1_name),
TextFileWriter(f2_name)],
check_format=True)
# Should raise an error if formats are not compatible
with self.assertLogs(logging.getLogger(), logging.ERROR):
with self.assertRaises(ValueError):
ComposedWriter(
transforms=[ParseNMEATransform()],
writers=[TextFileWriter(f1_name),
TextFileWriter(f2_name)],
check_format=True)
def test_default(self):
p = ParseNMEATransform()
q = QCFilterTransform(bounds='Grav1ValueMg:22000:23000,Grav1Error::2')
record = p.transform('grv1 2017-11-04:05:12:21.273413 01:022013 00')
self.assertIsNone(q.transform(record))
record = p.transform('grv1 2017-11-04:05:12:21.273413 01:022013 -5')
self.assertIsNone(q.transform(record))
record = p.transform('grv1 2017-11-04:05:12:21.273413 01:023013 00')
self.assertEqual(q.transform(record),
'Grav1ValueMg: 23013 > upper bound 23000')
record = p.transform('grv1 2017-11-04:05:12:21.273413 01:023013 03')
self.assertEqual(q.transform(record).split(';').sort(),
'Grav1ValueMg: 23013 > upper bound 23000; Grav1Error: 3 > upper bound 2'.split(';').sort())
##########################
# Transforms
if new_args.slice:
transforms.append(
SliceTransform(new_args.slice, all_args.slice_separator))
if new_args.timestamp:
transforms.append(TimestampTransform())
if new_args.prefix:
transforms.append(PrefixTransform(new_args.prefix))
if new_args.regex_filter:
transforms.append(RegexFilterTransform(new_args.regex_filter))
if new_args.qc_filter:
transforms.append(QCFilterTransform(new_args.qc_filter))
if new_args.parse_nmea:
transforms.append(ParseNMEATransform())
##########################
# Writers
if new_args.write_file:
for filename in new_args.write_file.split(','):
if filename == '-':
filename = None
writers.append(TextFileWriter(filename=filename))
if new_args.write_logfile:
writers.append(LogfileWriter(filebase=new_args.write_logfile))
if new_args.write_network:
for addr in new_args.write_network.split(','):
writers.append(NetworkWriter(network=addr))
if new_args.write_record_screen:
writers.append(RecordScreenWriter())
# Transforms
if new_args.slice:
transforms.append(
SliceTransform(new_args.slice, all_args.slice_separator))
if new_args.timestamp:
transforms.append(
TimestampTransform(time_format=all_args.time_format))
if new_args.prefix:
transforms.append(PrefixTransform(new_args.prefix))
if new_args.regex_filter:
transforms.append(RegexFilterTransform(new_args.regex_filter))
if new_args.qc_filter:
transforms.append(QCFilterTransform(new_args.qc_filter))
if new_args.parse_nmea:
transforms.append(
ParseNMEATransform(time_format=all_args.time_format))
if new_args.aggregate_xml:
transforms.append(
XMLAggregatorTransform(new_args.aggregate_xml))
##########################
# Writers
if new_args.write_file:
for filename in new_args.write_file.split(','):
if filename == '-':
filename = None
writers.append(TextFileWriter(filename=filename))
if new_args.write_logfile:
writers.append(LogfileWriter(filebase=new_args.write_logfile))
if new_args.write_network:
for addr in new_args.write_network.split(','):
all_args.slice_separator))
if new_args.timestamp:
transforms.append(TimestampTransform(time_format=all_args.time_format))
if new_args.prefix:
transforms.append(PrefixTransform(new_args.prefix))
if new_args.extract:
transforms.append(ExtractFieldTransform(new_args.extract))
if new_args.regex_filter:
transforms.append(RegexFilterTransform(new_args.regex_filter))
if new_args.qc_filter:
transforms.append(QCFilterTransform(new_args.qc_filter))
if new_args.parse_nmea:
transforms.append(
ParseNMEATransform(
message_path=all_args.parse_nmea_message_path,
sensor_path=all_args.parse_nmea_sensor_path,
sensor_model_path=all_args.parse_nmea_sensor_model_path,
time_format=all_args.time_format)
)
if new_args.parse:
transforms.append(
ParseTransform(
definition_path=all_args.parse_definition_path,
return_json=all_args.parse_to_json,
return_das_record=all_args.parse_to_das_record)
)
if new_args.aggregate_xml:
transforms.append(XMLAggregatorTransform(new_args.aggregate_xml))
if new_args.max_min:
transforms.append(MaxMinTransform())