def test_accel_mag_no_temp(self):
full_file_path = reference_file('accel_mag_no_temp.lid')
parameters = default_parameters()
parameters['average'] = False
dc = DataConverter(full_file_path, parameters)
dc.convert()
assert_compare_expected_file('accel_mag_no_temp_AccelMag.csv')
def test_current_no_accel(self):
full_file_path = reference_file('temp_mag_no_accel.lid')
parameters = default_parameters()
parameters['output_type'] = 'current'
dc = DataConverter(full_file_path, parameters)
with self.assertRaises(ValueError):
dc.convert()
def test_compass(self):
full_file_path = reference_file('test.lid')
parameters = default_parameters()
parameters['output_type'] = 'compass'
dc = DataConverter(full_file_path, parameters)
dc.convert()
compare_files(reference_file('test_Heading.csv'),
reference_file('test_Heading_GS.txt'))
def test_conversion(self):
full_file_path = reference_file('test.lid')
parameters = default_parameters()
parameters['average'] = False
converter = DataConverter(full_file_path, parameters)
converter.convert()
assert_compare_expected_file('test_AccelMag.csv')
assert_compare_expected_file('test_Temperature.csv')
def test_yaw_pitch_roll(self):
full_file_path = reference_file('test.lid')
parameters = default_parameters()
parameters['output_type'] = 'ypr'
dc = DataConverter(full_file_path, parameters)
dc.convert()
compare_files(reference_file('test_YawPitchRoll.csv'),
reference_file('test_ypr_GS.txt'))
def test_observer(self):
full_file_path = reference_file('test.lid')
parameters = default_parameters()
parameters['average'] = False
dc = DataConverter(full_file_path, parameters)
dc.register_observer(lambda percent_done: None)
dc.convert()
assert_compare_expected_file('test_AccelMag.csv')
assert_compare_expected_file('test_Temperature.csv')
def test_convert_w_posix_time(self):
full_file_path = reference_file('test.lid')
parameters = default_parameters()
parameters['time_format'] = 'posix'
dc = DataConverter(full_file_path, parameters)
dc.convert()
assert_compare_expected_file('test_AccelMag.csv',
'test_AccelMag-posix.csv.expect')
assert_compare_expected_file('test_Temperature.csv',
'test_Temperature-posix.csv.expect')
def test_specify_file_name(self):
full_file_path = reference_file('test.lid')
parameters = default_parameters()
parameters['file_name'] = 'calley'
dc = DataConverter(full_file_path, parameters)
dc.convert()
compare_files(reference_file('calley_AccelMag.csv'),
reference_file('test_AccelMag-posix.csv.expect'))
compare_files(reference_file('calley_Temperature.csv'),
reference_file('test_Temperature-posix.csv.expect'))
def test_current(self):
full_file_path = reference_file('test.lid')
tilt_file_path = reference_file('tiltcurve/TCM-1, 1BalSalt.cal')
tilt_curve = TiltCurve(tilt_file_path)
parameters = default_parameters()
parameters['output_type'] = 'current'
parameters['tilt_curve'] = tilt_curve
dc = DataConverter(full_file_path, parameters)
dc.convert()
assert_compare_expected_file('test_Current.csv')
assert_compare_expected_file('test_Temperature.csv')
def test_custom_calibration(self):
full_file_path = reference_file('custom_cal/test.lid')
cal_path = reference_file('custom_cal/hoststorage_default.txt')
calibration = make_from_calibration_file(cal_path)
parameters = default_parameters()
parameters['calibration'] = calibration
parameters['average'] = False
dc = DataConverter(full_file_path, parameters)
dc.convert()
assert_compare_expected_file('custom_cal/test_AccelMag.csv')
assert_compare_expected_file('custom_cal/test_Temperature.csv')
def test_current_with_spike(self):
full_file_path = reference_file('spike.lid')
tilt_file_path = reference_file(
'tiltcurve/TCM-3, Deep Water Meter.cal')
tilt_curve = TiltCurve(tilt_file_path)
parameters = default_parameters()
parameters['output_type'] = 'current'
parameters['tilt_curve'] = tilt_curve
dc = DataConverter(full_file_path, parameters)
dc.convert()
assert_compare_expected_file('spike_Current.csv')
assert_compare_expected_file('spike_Temperature.csv')
def utils_mat_convert_data(data, path, size):
if data == b'':
return False
try:
with open(path, 'wb') as f:
f.write(data)
f.truncate(size)
pars = default_parameters()
converter = DataConverter(path, pars)
converter.convert()
return True
except Exception as ex:
print(ex)
return False
class FileConverter(QThread):
progress_signal = pyqtSignal(int, int)
conversion_status_signal = pyqtSignal(str, int, int)
file_converted_signal = pyqtSignal()
conversion_complete = pyqtSignal()
ask_overwrite_signal = pyqtSignal(str)
def __init__(self, data_file_container, parameters):
# parameters is a dict of parameters required by FileConverter
super().__init__()
self.mutex = QMutex()
self.wait_condition = QWaitCondition()
self.data_file_container = data_file_container
self.parameters = parameters
self.current_file_ind = 0
self.file_sizes = [file.size if file.status == 'unconverted' else 0
for file in data_file_container]
self.unconverted_files = self.data_file_container.unconverted()
self.total_mb = sum(self.file_sizes)
self._is_running = False
self.converter = None
self.overwrite = None
def run(self):
self._is_running = True
count = 0
for i, file in enumerate(self.data_file_container):
if not self._is_running:
break
if file.status != 'unconverted':
continue
count += 1
self.current_file_ind = i
self.conversion_status_signal.emit(
file.filename, count, self.unconverted_files)
# if not os.path.isfile(file.path):
# file.status = 'not found'
# continue
self._convert_file(file)
self.file_converted_signal.emit()
self.conversion_complete.emit()
def _convert_file(self, file):
self.current_file = file
try:
conversion_parameters = default_parameters()
conversion_parameters.update(self.parameters)
conversion_parameters['overwrite'] = self._process_overwrite()
self.converter = DataConverter(file.path, conversion_parameters)
self.converter.register_observer(self.update_progress)
self.converter.convert()
self.data_file_container.change_status(file, 'converted')
except FileExistsError as message:
self.ask_overwrite(file.filename)
if self.overwrite in ['once', 'yes_to_all']:
self._convert_file(file)
except (TypeError, ValueError, IndexError) as m:
if str(m) == 'Not all required sensors present':
self.data_file_container.change_status(
file, 'error_sensor_missing')
else:
self.data_file_container.change_status(file, 'error_failed')
except FileNotFoundError:
self.data_file_container.change_status(file, 'error_missing')
except NoDataError:
self.data_file_container.change_status(file, 'error_no_data')
finally:
self.converter.close_source()
# check for the case that the conversion was canceled
if not self.converter._is_running:
self.data_file_container.change_status(file, 'unconverted')
def update_progress(self, percent_done):
# This is an observer function that gets notified when a data
# page is parsed
if not self._is_running:
self.converter.cancel_conversion()
cumulative_mb = sum([size for size
in self.file_sizes[:self.current_file_ind]])
cumulative_mb += (self.data_file_container[self.current_file_ind].size
* (percent_done/100))
overall_percent = cumulative_mb / self.total_mb
overall_percent *= 100
self.progress_signal.emit(percent_done, overall_percent)
def _process_overwrite(self):
# TODO why is there a tuple used below when index 0 isn't used?
action_map = {
'once': (None, True),
'yes_to_all': ('yes_to_all', True),
'no': (None, False),
'no_to_all': ('no_to_all', False)
}
_, overwrite_status = action_map.get(self.overwrite, (None, False))
return overwrite_status
def ask_overwrite(self, filename):
if self.overwrite is None:
self.ask_overwrite_signal.emit(str(filename))
self.mutex.lock()
self.wait_condition.wait(self.mutex)
self.mutex.unlock()
def set_overwrite(self, state):
self.overwrite = state
self.wait_condition.wakeAll()
def cancel(self):
self._is_running = False
def test_temp_comp_magnetometer(self):
full_file_path = reference_file('TCM1_Calibrate_(0).lid')
dc = DataConverter(full_file_path, default_parameters())
dc.convert()
assert_compare_expected_file('TCM1_Calibrate_(0)_AccelMag.csv')
assert_compare_expected_file('TCM1_Calibrate_(0)_Temperature.csv')