def test_current_no_accel(self): full_file_path = reference_file('temp_mag_no_accel.lid') parameters = default_parameters() parameters['output_type'] = 'current' dc = DataConverter(full_file_path, parameters) with self.assertRaises(ValueError): dc.convert()
def test_accel_mag_no_temp(self): full_file_path = reference_file('accel_mag_no_temp.lid') parameters = default_parameters() parameters['average'] = False dc = DataConverter(full_file_path, parameters) dc.convert() assert_compare_expected_file('accel_mag_no_temp_AccelMag.csv')
def _read_output_options(self): parameters = default_parameters() app_data = self.settings.value('output_options', {}, type=dict) parameters['output_directory'] = self._get_output_directory() parameters['time_format'] = app_data.get('time_format', 'iso8601') parameters['average'] = app_data.get('average_bursts', True) custom_cal_path = app_data.get('custom_cal', None) if custom_cal_path: try: custom_cal = make_from_calibration_file(custom_cal_path) parameters['calibration'] = custom_cal except: pass split_size = app_data.get('split', 'Do not split output files') if split_size != 'Do not split output files': parameters['split'] = int(split_size.split(' ')[0]) if self.comboBox_output_type.currentText() == 'Current': parameters['output_type'] = 'current' parameters['tilt_curve'] = \ self.comboBox_tilt_tables.currentData() else: output_type = OUTPUT_TYPE.get( self.comboBox_output_type.currentText(), 'discrete') parameters['output_type'] = output_type parameters['output_format'] = app_data.get('output_format', 'csv') parameters['declination'] = self.dec_model.declination_value() return parameters
def _convert_file(self, file): self.current_file = file try: conversion_parameters = default_parameters() conversion_parameters.update(self.parameters) conversion_parameters['overwrite'] = self._process_overwrite() self.converter = DataConverter(file.path, conversion_parameters) self.converter.register_observer(self.update_progress) self.converter.convert() self.data_file_container.change_status(file, 'converted') except FileExistsError as message: self.ask_overwrite(file.filename) if self.overwrite in ['once', 'yes_to_all']: self._convert_file(file) except (TypeError, ValueError, IndexError) as m: if str(m) == 'Not all required sensors present': self.data_file_container.change_status( file, 'error_sensor_missing') else: self.data_file_container.change_status(file, 'error_failed') except FileNotFoundError: self.data_file_container.change_status(file, 'error_missing') except NoDataError: self.data_file_container.change_status(file, 'error_no_data') finally: self.converter.close_source() # check for the case that the conversion was canceled if not self.converter._is_running: self.data_file_container.change_status(file, 'unconverted')
def test_conversion(self): full_file_path = reference_file('test.lid') parameters = default_parameters() parameters['average'] = False converter = DataConverter(full_file_path, parameters) converter.convert() assert_compare_expected_file('test_AccelMag.csv') assert_compare_expected_file('test_Temperature.csv')
def test_compass(self): full_file_path = reference_file('test.lid') parameters = default_parameters() parameters['output_type'] = 'compass' dc = DataConverter(full_file_path, parameters) dc.convert() compare_files(reference_file('test_Heading.csv'), reference_file('test_Heading_GS.txt'))
def test_yaw_pitch_roll(self): full_file_path = reference_file('test.lid') parameters = default_parameters() parameters['output_type'] = 'ypr' dc = DataConverter(full_file_path, parameters) dc.convert() compare_files(reference_file('test_YawPitchRoll.csv'), reference_file('test_ypr_GS.txt'))
def test_observer(self): full_file_path = reference_file('test.lid') parameters = default_parameters() parameters['average'] = False dc = DataConverter(full_file_path, parameters) dc.register_observer(lambda percent_done: None) dc.convert() assert_compare_expected_file('test_AccelMag.csv') assert_compare_expected_file('test_Temperature.csv')
def test_specify_file_name(self): full_file_path = reference_file('test.lid') parameters = default_parameters() parameters['file_name'] = 'calley' dc = DataConverter(full_file_path, parameters) dc.convert() compare_files(reference_file('calley_AccelMag.csv'), reference_file('test_AccelMag-posix.csv.expect')) compare_files(reference_file('calley_Temperature.csv'), reference_file('test_Temperature-posix.csv.expect'))
def test_convert_w_posix_time(self): full_file_path = reference_file('test.lid') parameters = default_parameters() parameters['time_format'] = 'posix' dc = DataConverter(full_file_path, parameters) dc.convert() assert_compare_expected_file('test_AccelMag.csv', 'test_AccelMag-posix.csv.expect') assert_compare_expected_file('test_Temperature.csv', 'test_Temperature-posix.csv.expect')
def test_custom_calibration(self): full_file_path = reference_file('custom_cal/test.lid') cal_path = reference_file('custom_cal/hoststorage_default.txt') calibration = make_from_calibration_file(cal_path) parameters = default_parameters() parameters['calibration'] = calibration parameters['average'] = False dc = DataConverter(full_file_path, parameters) dc.convert() assert_compare_expected_file('custom_cal/test_AccelMag.csv') assert_compare_expected_file('custom_cal/test_Temperature.csv')
def test_current(self): full_file_path = reference_file('test.lid') tilt_file_path = reference_file('tiltcurve/TCM-1, 1BalSalt.cal') tilt_curve = TiltCurve(tilt_file_path) parameters = default_parameters() parameters['output_type'] = 'current' parameters['tilt_curve'] = tilt_curve dc = DataConverter(full_file_path, parameters) dc.convert() assert_compare_expected_file('test_Current.csv') assert_compare_expected_file('test_Temperature.csv')
def test_current_with_spike(self): full_file_path = reference_file('spike.lid') tilt_file_path = reference_file( 'tiltcurve/TCM-3, Deep Water Meter.cal') tilt_curve = TiltCurve(tilt_file_path) parameters = default_parameters() parameters['output_type'] = 'current' parameters['tilt_curve'] = tilt_curve dc = DataConverter(full_file_path, parameters) dc.convert() assert_compare_expected_file('spike_Current.csv') assert_compare_expected_file('spike_Temperature.csv')
def utils_mat_convert_data(data, path, size): if data == b'': return False try: with open(path, 'wb') as f: f.write(data) f.truncate(size) pars = default_parameters() converter = DataConverter(path, pars) converter.convert() return True except Exception as ex: print(ex) return False
def lid_to_csv(fol, suffix, sig, files_to_ignore=[]) -> (bool, list): """ converts depending on fileX_suffix.lid is an existing file """ # check asked MAC folder exists err_files = [] if not Path(fol).is_dir(): sig.error.emit('error: {} does not exist'.format(fol)) return False, err_files # check asked sensor data type exists valid_suffixes = ('_DissolvedOxygen', '_Temperature', '_Pressure') if suffix not in valid_suffixes: sig.error.emit('error: asked unknown suffix {}'.format(suffix)) return False, err_files # conversion parameters = default_parameters() lid_files = linux_ls_by_ext(fol, 'lid') all_ok = True for f in lid_files: # prevents continuous warnings on same files if f in files_to_ignore: continue # skip already converted files _ = '{}{}.csv'.format(f.split('.')[0], suffix) if Path(_).is_file(): # print('>> file {} already exists'.format(_)) continue # skip files not containing this sensor data if not _lid_file_has_sensor_data_type(f, suffix): sig.debug.emit('SYS: file {} has no {} data'.format(f, suffix)) continue # try to convert try: DataConverter(f, parameters).convert() sig.debug.emit('SYS: OK -> {}, suffix {}'.format(f, suffix)) except (ValueError, Exception) as ve: all_ok = False err_files.append(f) sig.error.emit('SYS: error {} -> {}'.format(f, ve)) return all_ok, err_files
def test_parameters(input, types, params, new_ui, qtbot, mocker): file = full_path(input) all_params = default_parameters() all_params.update(params) file_mock = mocker.patch('gui.converter_window.dialogs.open_lid_file') file_mock.return_value = [[str(file)], None] new_ui._read_output_options = lambda: all_params with qtbot.waitSignal(new_ui.file_loader.finished_signal, timeout=1000): qtbot.mouseClick(new_ui.pushButton_add, Qt.LeftButton) complete_signal = new_ui.converter.conversion_complete with qtbot.waitSignal(complete_signal, timeout=4000) as blocker: # ui.convert_files() qtbot.mouseClick(new_ui.pushButton_convert, Qt.LeftButton) for t in types: expect = expected_filename(input, t, all_params) converted = '{}_{}.csv'.format(input[:-4], t) compare(expect, converted)
def test_temp_comp_magnetometer(self): full_file_path = reference_file('TCM1_Calibrate_(0).lid') dc = DataConverter(full_file_path, default_parameters()) dc.convert() assert_compare_expected_file('TCM1_Calibrate_(0)_AccelMag.csv') assert_compare_expected_file('TCM1_Calibrate_(0)_Temperature.csv')
def test_unsupported_format(self): full_file_path = reference_file('test.lid') parameters = default_parameters() parameters['output_format'] = 'unsupported' with self.assertRaises(ValueError): DataConverter(full_file_path, parameters).convert()
def test_data_converter_creation(self): full_file_path = reference_file("test.lid") parameters = default_parameters() parameters['output_format'] = 'csv' DataConverter(full_file_path, parameters)
import glob import os from mat.data_converter import default_parameters, DataConverter # place this script inside a DDH folder along with any .lid files to be converted # $ cd /home/pi/Desktop/<your_folder> # $ sudo python3 main_cnv.py # this script will not convert already converted files if __name__ == '__main__': fol = str(os.getcwd()) # fol = '/home/joaquim/Desktop/04-ee-03-73-87-24' parameters = default_parameters() wc = fol + '/**/*.lid' lid_files = glob.glob(wc, recursive=True) wc = fol + '/**/*.csv' csv_files = glob.glob(wc, recursive=True) for f in lid_files: # build file name .csv _ = '{}_DissolvedOxygen.csv'.format(f.split('.')[0]) if _ not in csv_files: try: # converting takes about 1.5 seconds per file DataConverter(f, parameters).convert() s = 'file {} conversion OK'.format(f) print(s) except (ValueError, Exception) as ve: e = 'file {} ERROR conversion -> {}'.format(f, ve)
def test_creation(self): assert DataConverter('no file', default_parameters())