def test_current_no_accel(self):
full_file_path = reference_file('temp_mag_no_accel.lid')
parameters = default_parameters()
parameters['output_type'] = 'current'
dc = DataConverter(full_file_path, parameters)
with self.assertRaises(ValueError):
dc.convert()
def test_accel_mag_no_temp(self):
full_file_path = reference_file('accel_mag_no_temp.lid')
parameters = default_parameters()
parameters['average'] = False
dc = DataConverter(full_file_path, parameters)
dc.convert()
assert_compare_expected_file('accel_mag_no_temp_AccelMag.csv')
def _read_output_options(self):
parameters = default_parameters()
app_data = self.settings.value('output_options', {}, type=dict)
parameters['output_directory'] = self._get_output_directory()
parameters['time_format'] = app_data.get('time_format', 'iso8601')
parameters['average'] = app_data.get('average_bursts', True)
custom_cal_path = app_data.get('custom_cal', None)
if custom_cal_path:
try:
custom_cal = make_from_calibration_file(custom_cal_path)
parameters['calibration'] = custom_cal
except:
pass
split_size = app_data.get('split', 'Do not split output files')
if split_size != 'Do not split output files':
parameters['split'] = int(split_size.split(' ')[0])
if self.comboBox_output_type.currentText() == 'Current':
parameters['output_type'] = 'current'
parameters['tilt_curve'] = \
self.comboBox_tilt_tables.currentData()
else:
output_type = OUTPUT_TYPE.get(
self.comboBox_output_type.currentText(),
'discrete')
parameters['output_type'] = output_type
parameters['output_format'] = app_data.get('output_format', 'csv')
parameters['declination'] = self.dec_model.declination_value()
return parameters
def _convert_file(self, file):
self.current_file = file
try:
conversion_parameters = default_parameters()
conversion_parameters.update(self.parameters)
conversion_parameters['overwrite'] = self._process_overwrite()
self.converter = DataConverter(file.path, conversion_parameters)
self.converter.register_observer(self.update_progress)
self.converter.convert()
self.data_file_container.change_status(file, 'converted')
except FileExistsError as message:
self.ask_overwrite(file.filename)
if self.overwrite in ['once', 'yes_to_all']:
self._convert_file(file)
except (TypeError, ValueError, IndexError) as m:
if str(m) == 'Not all required sensors present':
self.data_file_container.change_status(
file, 'error_sensor_missing')
else:
self.data_file_container.change_status(file, 'error_failed')
except FileNotFoundError:
self.data_file_container.change_status(file, 'error_missing')
except NoDataError:
self.data_file_container.change_status(file, 'error_no_data')
finally:
self.converter.close_source()
# check for the case that the conversion was canceled
if not self.converter._is_running:
self.data_file_container.change_status(file, 'unconverted')
def test_conversion(self):
full_file_path = reference_file('test.lid')
parameters = default_parameters()
parameters['average'] = False
converter = DataConverter(full_file_path, parameters)
converter.convert()
assert_compare_expected_file('test_AccelMag.csv')
assert_compare_expected_file('test_Temperature.csv')
def test_compass(self):
full_file_path = reference_file('test.lid')
parameters = default_parameters()
parameters['output_type'] = 'compass'
dc = DataConverter(full_file_path, parameters)
dc.convert()
compare_files(reference_file('test_Heading.csv'),
reference_file('test_Heading_GS.txt'))
def test_yaw_pitch_roll(self):
full_file_path = reference_file('test.lid')
parameters = default_parameters()
parameters['output_type'] = 'ypr'
dc = DataConverter(full_file_path, parameters)
dc.convert()
compare_files(reference_file('test_YawPitchRoll.csv'),
reference_file('test_ypr_GS.txt'))
def test_observer(self):
full_file_path = reference_file('test.lid')
parameters = default_parameters()
parameters['average'] = False
dc = DataConverter(full_file_path, parameters)
dc.register_observer(lambda percent_done: None)
dc.convert()
assert_compare_expected_file('test_AccelMag.csv')
assert_compare_expected_file('test_Temperature.csv')
def test_specify_file_name(self):
full_file_path = reference_file('test.lid')
parameters = default_parameters()
parameters['file_name'] = 'calley'
dc = DataConverter(full_file_path, parameters)
dc.convert()
compare_files(reference_file('calley_AccelMag.csv'),
reference_file('test_AccelMag-posix.csv.expect'))
compare_files(reference_file('calley_Temperature.csv'),
reference_file('test_Temperature-posix.csv.expect'))
def test_convert_w_posix_time(self):
full_file_path = reference_file('test.lid')
parameters = default_parameters()
parameters['time_format'] = 'posix'
dc = DataConverter(full_file_path, parameters)
dc.convert()
assert_compare_expected_file('test_AccelMag.csv',
'test_AccelMag-posix.csv.expect')
assert_compare_expected_file('test_Temperature.csv',
'test_Temperature-posix.csv.expect')
def test_custom_calibration(self):
full_file_path = reference_file('custom_cal/test.lid')
cal_path = reference_file('custom_cal/hoststorage_default.txt')
calibration = make_from_calibration_file(cal_path)
parameters = default_parameters()
parameters['calibration'] = calibration
parameters['average'] = False
dc = DataConverter(full_file_path, parameters)
dc.convert()
assert_compare_expected_file('custom_cal/test_AccelMag.csv')
assert_compare_expected_file('custom_cal/test_Temperature.csv')
def test_current(self):
full_file_path = reference_file('test.lid')
tilt_file_path = reference_file('tiltcurve/TCM-1, 1BalSalt.cal')
tilt_curve = TiltCurve(tilt_file_path)
parameters = default_parameters()
parameters['output_type'] = 'current'
parameters['tilt_curve'] = tilt_curve
dc = DataConverter(full_file_path, parameters)
dc.convert()
assert_compare_expected_file('test_Current.csv')
assert_compare_expected_file('test_Temperature.csv')
def test_current_with_spike(self):
full_file_path = reference_file('spike.lid')
tilt_file_path = reference_file(
'tiltcurve/TCM-3, Deep Water Meter.cal')
tilt_curve = TiltCurve(tilt_file_path)
parameters = default_parameters()
parameters['output_type'] = 'current'
parameters['tilt_curve'] = tilt_curve
dc = DataConverter(full_file_path, parameters)
dc.convert()
assert_compare_expected_file('spike_Current.csv')
assert_compare_expected_file('spike_Temperature.csv')
def utils_mat_convert_data(data, path, size):
if data == b'':
return False
try:
with open(path, 'wb') as f:
f.write(data)
f.truncate(size)
pars = default_parameters()
converter = DataConverter(path, pars)
converter.convert()
return True
except Exception as ex:
print(ex)
return False
def lid_to_csv(fol, suffix, sig, files_to_ignore=[]) -> (bool, list):
""" converts depending on fileX_suffix.lid is an existing file """
# check asked MAC folder exists
err_files = []
if not Path(fol).is_dir():
sig.error.emit('error: {} does not exist'.format(fol))
return False, err_files
# check asked sensor data type exists
valid_suffixes = ('_DissolvedOxygen', '_Temperature', '_Pressure')
if suffix not in valid_suffixes:
sig.error.emit('error: asked unknown suffix {}'.format(suffix))
return False, err_files
# conversion
parameters = default_parameters()
lid_files = linux_ls_by_ext(fol, 'lid')
all_ok = True
for f in lid_files:
# prevents continuous warnings on same files
if f in files_to_ignore:
continue
# skip already converted files
_ = '{}{}.csv'.format(f.split('.')[0], suffix)
if Path(_).is_file():
# print('>> file {} already exists'.format(_))
continue
# skip files not containing this sensor data
if not _lid_file_has_sensor_data_type(f, suffix):
sig.debug.emit('SYS: file {} has no {} data'.format(f, suffix))
continue
# try to convert
try:
DataConverter(f, parameters).convert()
sig.debug.emit('SYS: OK -> {}, suffix {}'.format(f, suffix))
except (ValueError, Exception) as ve:
all_ok = False
err_files.append(f)
sig.error.emit('SYS: error {} -> {}'.format(f, ve))
return all_ok, err_files
def test_parameters(input, types, params, new_ui, qtbot, mocker):
file = full_path(input)
all_params = default_parameters()
all_params.update(params)
file_mock = mocker.patch('gui.converter_window.dialogs.open_lid_file')
file_mock.return_value = [[str(file)], None]
new_ui._read_output_options = lambda: all_params
with qtbot.waitSignal(new_ui.file_loader.finished_signal, timeout=1000):
qtbot.mouseClick(new_ui.pushButton_add, Qt.LeftButton)
complete_signal = new_ui.converter.conversion_complete
with qtbot.waitSignal(complete_signal, timeout=4000) as blocker:
# ui.convert_files()
qtbot.mouseClick(new_ui.pushButton_convert, Qt.LeftButton)
for t in types:
expect = expected_filename(input, t, all_params)
converted = '{}_{}.csv'.format(input[:-4], t)
compare(expect, converted)
def test_temp_comp_magnetometer(self):
full_file_path = reference_file('TCM1_Calibrate_(0).lid')
dc = DataConverter(full_file_path, default_parameters())
dc.convert()
assert_compare_expected_file('TCM1_Calibrate_(0)_AccelMag.csv')
assert_compare_expected_file('TCM1_Calibrate_(0)_Temperature.csv')
def test_unsupported_format(self):
full_file_path = reference_file('test.lid')
parameters = default_parameters()
parameters['output_format'] = 'unsupported'
with self.assertRaises(ValueError):
DataConverter(full_file_path, parameters).convert()
def test_data_converter_creation(self):
full_file_path = reference_file("test.lid")
parameters = default_parameters()
parameters['output_format'] = 'csv'
DataConverter(full_file_path, parameters)
import glob
import os
from mat.data_converter import default_parameters, DataConverter
# place this script inside a DDH folder along with any .lid files to be converted
# $ cd /home/pi/Desktop/<your_folder>
# $ sudo python3 main_cnv.py
# this script will not convert already converted files
if __name__ == '__main__':
fol = str(os.getcwd())
# fol = '/home/joaquim/Desktop/04-ee-03-73-87-24'
parameters = default_parameters()
wc = fol + '/**/*.lid'
lid_files = glob.glob(wc, recursive=True)
wc = fol + '/**/*.csv'
csv_files = glob.glob(wc, recursive=True)
for f in lid_files:
# build file name .csv
_ = '{}_DissolvedOxygen.csv'.format(f.split('.')[0])
if _ not in csv_files:
try:
# converting takes about 1.5 seconds per file
DataConverter(f, parameters).convert()
s = 'file {} conversion OK'.format(f)
print(s)
except (ValueError, Exception) as ve:
e = 'file {} ERROR conversion -> {}'.format(f, ve)
def test_creation(self):
assert DataConverter('no file', default_parameters())