def test_current_no_accel(self):
full_file_path = reference_file('temp_mag_no_accel.lid')
parameters = default_parameters()
parameters['output_type'] = 'current'
dc = DataConverter(full_file_path, parameters)
with self.assertRaises(ValueError):
dc.convert()
def test_accel_mag_no_temp(self):
full_file_path = reference_file('accel_mag_no_temp.lid')
parameters = default_parameters()
parameters['average'] = False
dc = DataConverter(full_file_path, parameters)
dc.convert()
assert_compare_expected_file('accel_mag_no_temp_AccelMag.csv')
def _convert_file(self, file):
self.current_file = file
try:
conversion_parameters = default_parameters()
conversion_parameters.update(self.parameters)
conversion_parameters['overwrite'] = self._process_overwrite()
self.converter = DataConverter(file.path, conversion_parameters)
self.converter.register_observer(self.update_progress)
self.converter.convert()
self.data_file_container.change_status(file, 'converted')
except FileExistsError as message:
self.ask_overwrite(file.filename)
if self.overwrite in ['once', 'yes_to_all']:
self._convert_file(file)
except (TypeError, ValueError, IndexError) as m:
if str(m) == 'Not all required sensors present':
self.data_file_container.change_status(
file, 'error_sensor_missing')
else:
self.data_file_container.change_status(file, 'error_failed')
except FileNotFoundError:
self.data_file_container.change_status(file, 'error_missing')
except NoDataError:
self.data_file_container.change_status(file, 'error_no_data')
finally:
self.converter.close_source()
# check for the case that the conversion was canceled
if not self.converter._is_running:
self.data_file_container.change_status(file, 'unconverted')
def test_conversion(self):
full_file_path = reference_file('test.lid')
parameters = default_parameters()
parameters['average'] = False
converter = DataConverter(full_file_path, parameters)
converter.convert()
assert_compare_expected_file('test_AccelMag.csv')
assert_compare_expected_file('test_Temperature.csv')
def test_yaw_pitch_roll(self):
full_file_path = reference_file('test.lid')
parameters = default_parameters()
parameters['output_type'] = 'ypr'
dc = DataConverter(full_file_path, parameters)
dc.convert()
compare_files(reference_file('test_YawPitchRoll.csv'),
reference_file('test_ypr_GS.txt'))
def test_compass(self):
full_file_path = reference_file('test.lid')
parameters = default_parameters()
parameters['output_type'] = 'compass'
dc = DataConverter(full_file_path, parameters)
dc.convert()
compare_files(reference_file('test_Heading.csv'),
reference_file('test_Heading_GS.txt'))
def test_convert_w_posix_time(self):
full_file_path = reference_file('test.lid')
parameters = default_parameters()
parameters['time_format'] = 'posix'
dc = DataConverter(full_file_path, parameters)
dc.convert()
assert_compare_expected_file('test_AccelMag.csv',
'test_AccelMag-posix.csv.expect')
assert_compare_expected_file('test_Temperature.csv',
'test_Temperature-posix.csv.expect')
def test_specify_file_name(self):
full_file_path = reference_file('test.lid')
parameters = default_parameters()
parameters['file_name'] = 'calley'
dc = DataConverter(full_file_path, parameters)
dc.convert()
compare_files(reference_file('calley_AccelMag.csv'),
reference_file('test_AccelMag-posix.csv.expect'))
compare_files(reference_file('calley_Temperature.csv'),
reference_file('test_Temperature-posix.csv.expect'))
def test_current(self):
full_file_path = reference_file('test.lid')
tilt_file_path = reference_file('tiltcurve/TCM-1, 1BalSalt.cal')
tilt_curve = TiltCurve(tilt_file_path)
parameters = default_parameters()
parameters['output_type'] = 'current'
parameters['tilt_curve'] = tilt_curve
dc = DataConverter(full_file_path, parameters)
dc.convert()
assert_compare_expected_file('test_Current.csv')
assert_compare_expected_file('test_Temperature.csv')
def test_custom_calibration(self):
full_file_path = reference_file('custom_cal/test.lid')
cal_path = reference_file('custom_cal/hoststorage_default.txt')
calibration = make_from_calibration_file(cal_path)
parameters = default_parameters()
parameters['calibration'] = calibration
parameters['average'] = False
dc = DataConverter(full_file_path, parameters)
dc.convert()
assert_compare_expected_file('custom_cal/test_AccelMag.csv')
assert_compare_expected_file('custom_cal/test_Temperature.csv')
def test_current_with_spike(self):
full_file_path = reference_file('spike.lid')
tilt_file_path = reference_file(
'tiltcurve/TCM-3, Deep Water Meter.cal')
tilt_curve = TiltCurve(tilt_file_path)
parameters = default_parameters()
parameters['output_type'] = 'current'
parameters['tilt_curve'] = tilt_curve
dc = DataConverter(full_file_path, parameters)
dc.convert()
assert_compare_expected_file('spike_Current.csv')
assert_compare_expected_file('spike_Temperature.csv')
def utils_mat_convert_data(data, path, size):
if data == b'':
return False
try:
with open(path, 'wb') as f:
f.write(data)
f.truncate(size)
pars = default_parameters()
converter = DataConverter(path, pars)
converter.convert()
return True
except Exception as ex:
print(ex)
return False
def test_observer(self):
full_file_path = reference_file('test.lid')
parameters = default_parameters()
parameters['average'] = False
dc = DataConverter(full_file_path, parameters)
dc.register_observer(lambda percent_done: None)
dc.convert()
assert_compare_expected_file('test_AccelMag.csv')
assert_compare_expected_file('test_Temperature.csv')
def lid_to_csv(fol, suffix, sig, files_to_ignore=[]) -> (bool, list):
""" converts depending on fileX_suffix.lid is an existing file """
# check asked MAC folder exists
err_files = []
if not Path(fol).is_dir():
sig.error.emit('error: {} does not exist'.format(fol))
return False, err_files
# check asked sensor data type exists
valid_suffixes = ('_DissolvedOxygen', '_Temperature', '_Pressure')
if suffix not in valid_suffixes:
sig.error.emit('error: asked unknown suffix {}'.format(suffix))
return False, err_files
# conversion
parameters = default_parameters()
lid_files = linux_ls_by_ext(fol, 'lid')
all_ok = True
for f in lid_files:
# prevents continuous warnings on same files
if f in files_to_ignore:
continue
# skip already converted files
_ = '{}{}.csv'.format(f.split('.')[0], suffix)
if Path(_).is_file():
# print('>> file {} already exists'.format(_))
continue
# skip files not containing this sensor data
if not _lid_file_has_sensor_data_type(f, suffix):
sig.debug.emit('SYS: file {} has no {} data'.format(f, suffix))
continue
# try to convert
try:
DataConverter(f, parameters).convert()
sig.debug.emit('SYS: OK -> {}, suffix {}'.format(f, suffix))
except (ValueError, Exception) as ve:
all_ok = False
err_files.append(f)
sig.error.emit('SYS: error {} -> {}'.format(f, ve))
return all_ok, err_files
class FileConverter(QThread):
progress_signal = pyqtSignal(int, int)
conversion_status_signal = pyqtSignal(str, int, int)
file_converted_signal = pyqtSignal()
conversion_complete = pyqtSignal()
ask_overwrite_signal = pyqtSignal(str)
def __init__(self, data_file_container, parameters):
# parameters is a dict of parameters required by FileConverter
super().__init__()
self.mutex = QMutex()
self.wait_condition = QWaitCondition()
self.data_file_container = data_file_container
self.parameters = parameters
self.current_file_ind = 0
self.file_sizes = [file.size if file.status == 'unconverted' else 0
for file in data_file_container]
self.unconverted_files = self.data_file_container.unconverted()
self.total_mb = sum(self.file_sizes)
self._is_running = False
self.converter = None
self.overwrite = None
def run(self):
self._is_running = True
count = 0
for i, file in enumerate(self.data_file_container):
if not self._is_running:
break
if file.status != 'unconverted':
continue
count += 1
self.current_file_ind = i
self.conversion_status_signal.emit(
file.filename, count, self.unconverted_files)
# if not os.path.isfile(file.path):
# file.status = 'not found'
# continue
self._convert_file(file)
self.file_converted_signal.emit()
self.conversion_complete.emit()
def _convert_file(self, file):
self.current_file = file
try:
conversion_parameters = default_parameters()
conversion_parameters.update(self.parameters)
conversion_parameters['overwrite'] = self._process_overwrite()
self.converter = DataConverter(file.path, conversion_parameters)
self.converter.register_observer(self.update_progress)
self.converter.convert()
self.data_file_container.change_status(file, 'converted')
except FileExistsError as message:
self.ask_overwrite(file.filename)
if self.overwrite in ['once', 'yes_to_all']:
self._convert_file(file)
except (TypeError, ValueError, IndexError) as m:
if str(m) == 'Not all required sensors present':
self.data_file_container.change_status(
file, 'error_sensor_missing')
else:
self.data_file_container.change_status(file, 'error_failed')
except FileNotFoundError:
self.data_file_container.change_status(file, 'error_missing')
except NoDataError:
self.data_file_container.change_status(file, 'error_no_data')
finally:
self.converter.close_source()
# check for the case that the conversion was canceled
if not self.converter._is_running:
self.data_file_container.change_status(file, 'unconverted')
def update_progress(self, percent_done):
# This is an observer function that gets notified when a data
# page is parsed
if not self._is_running:
self.converter.cancel_conversion()
cumulative_mb = sum([size for size
in self.file_sizes[:self.current_file_ind]])
cumulative_mb += (self.data_file_container[self.current_file_ind].size
* (percent_done/100))
overall_percent = cumulative_mb / self.total_mb
overall_percent *= 100
self.progress_signal.emit(percent_done, overall_percent)
def _process_overwrite(self):
# TODO why is there a tuple used below when index 0 isn't used?
action_map = {
'once': (None, True),
'yes_to_all': ('yes_to_all', True),
'no': (None, False),
'no_to_all': ('no_to_all', False)
}
_, overwrite_status = action_map.get(self.overwrite, (None, False))
return overwrite_status
def ask_overwrite(self, filename):
if self.overwrite is None:
self.ask_overwrite_signal.emit(str(filename))
self.mutex.lock()
self.wait_condition.wait(self.mutex)
self.mutex.unlock()
def set_overwrite(self, state):
self.overwrite = state
self.wait_condition.wakeAll()
def cancel(self):
self._is_running = False
def test_temp_comp_magnetometer(self):
full_file_path = reference_file('TCM1_Calibrate_(0).lid')
dc = DataConverter(full_file_path, default_parameters())
dc.convert()
assert_compare_expected_file('TCM1_Calibrate_(0)_AccelMag.csv')
assert_compare_expected_file('TCM1_Calibrate_(0)_Temperature.csv')
def test_unsupported_format(self):
full_file_path = reference_file('test.lid')
parameters = default_parameters()
parameters['output_format'] = 'unsupported'
with self.assertRaises(ValueError):
DataConverter(full_file_path, parameters).convert()
def test_data_converter_creation(self):
full_file_path = reference_file("test.lid")
parameters = default_parameters()
parameters['output_format'] = 'csv'
DataConverter(full_file_path, parameters)
from mat.data_converter import default_parameters, DataConverter
# place this script inside a DDH folder along with any .lid files to be converted
# $ cd /home/pi/Desktop/<your_folder>
# $ sudo python3 main_cnv.py
# this script will not convert already converted files
if __name__ == '__main__':
fol = str(os.getcwd())
# fol = '/home/joaquim/Desktop/04-ee-03-73-87-24'
parameters = default_parameters()
wc = fol + '/**/*.lid'
lid_files = glob.glob(wc, recursive=True)
wc = fol + '/**/*.csv'
csv_files = glob.glob(wc, recursive=True)
for f in lid_files:
# build file name .csv
_ = '{}_DissolvedOxygen.csv'.format(f.split('.')[0])
if _ not in csv_files:
try:
# converting takes about 1.5 seconds per file
DataConverter(f, parameters).convert()
s = 'file {} conversion OK'.format(f)
print(s)
except (ValueError, Exception) as ve:
e = 'file {} ERROR conversion -> {}'.format(f, ve)
print(e)
def test_creation(self):
assert DataConverter('no file', default_parameters())
