def factor(self): value = self._factor.text() as_float = utils.to_float_or_none(value) if isinstance(as_float, float): return as_float else: return None
def factor(self): value = self._factor.text() as_float = utils.to_float_or_none(value) if isinstance(as_float, float): return as_float else: return None
def multiply_by_factor(file_data, columns_factors): """ Multiplies all values in the file data with a given factor for each column :param file_data: a list of lists like [['obsid', 'date_time', 'reading'], ['obs1', '2017-04-12 11:03', '123,456']] :param table_columns_factors: a dict like {'reading': 10} :return: file_data where the columns have been multiplied by the factor. """ file_data = [[str(float(col) * columns_factors[file_data[0][colnr]]) if (file_data[0][colnr] in columns_factors and utils.to_float_or_none(col) is not None) else col for colnr, col in enumerate(row)] if rownr > 0 else row for rownr, row in enumerate(file_data)] return file_data
def multiply_by_factor(file_data, columns_factors): """ Multiplies all values in the file data with a given factor for each column :param file_data: a list of lists like [['obsid', 'date_time', 'reading'], ['obs1', '2017-04-12 11:03', '123,456']] :param table_columns_factors: a dict like {'reading': 10} :return: file_data where the columns have been multiplied by the factor. """ file_data = [[str(float(col) * columns_factors[file_data[0][colnr]]) if (file_data[0][colnr] in columns_factors and utils.to_float_or_none(col) is not None) else col for colnr, col in enumerate(row)] if rownr > 0 else row for rownr, row in enumerate(file_data)] return file_data
def factor(self, value): as_float = utils.to_float_or_none(value) if isinstance(as_float, float): self._factor.setText(str(as_float))
def parse_levelogger_file(path, charset, skip_rows_without_water_level=False, begindate=None, enddate=None): """ Parses a levelogger csv file into a string :param path: The file name :param existing_obsids: A list or tuple with the obsids that exist in the db. :param ask_for_names: (True/False) True to ask for location name for every location. False to only ask if the location is not found in existing_obsids. :return: A string representing a table file. Including '\n' for newlines. Assumptions and limitations: * The Location attribute is used as location and added as a column. * Values containing ',' is replaced with '.' * Rows with missing "Water head[cm]"-data is skipped. """ #These can be set to paritally import files. #begindate = datetime.strptime('2016-06-08 20:00:00','%Y-%m-%d %H:%M:%S') #enddate = datetime.strptime('2016-06-08 19:00:00','%Y-%m-%d %H:%M:%S') #It should be possible to import all cols that exists in the translation dict filedata = [] location = None level_unit_factor_to_cm = 100 spec_cond_factor_to_mScm = 0.001 filename = os.path.basename(path) if begindate is not None: begindate = date_utils.datestring_to_date(begindate) if enddate is not None: enddate = date_utils.datestring_to_date(enddate) with io.open(path, 'rt', encoding=str(charset)) as f: rows_unsplit = [ row.lstrip().rstrip('\n').rstrip('\r') for row in f ] try: data_header_idx = [ rownr for rownr, row in enumerate(rows_unsplit) if row.startswith('Date') ][0] except IndexError: utils.MessagebarAndLog.warning(bar_msg=ru( QCoreApplication.translate( 'LeveloggerImport', '''File %s could not be parsed.''')) % filename) return [], filename, location delimiter = utils.get_delimiter_from_file_rows( rows_unsplit[data_header_idx:], filename=filename, delimiters=[';', ','], num_fields=None) if delimiter is None: return [], filename, location rows = [row.split(';') for row in rows_unsplit] lens = set([len(row) for row in rows[data_header_idx:]]) if len(lens) != 1 or list(lens)[0] == 1: # Assume that the delimiter was not ';' rows = [row.split(',') for row in rows_unsplit] col1 = [row[0] for row in rows] try: location_idx = col1.index('Location:') except ValueError: pass else: location = col1[location_idx + 1] try: level_unit_idx = col1.index('LEVEL') except ValueError: pass else: try: level_unit = col1[level_unit_idx + 1].split(':')[1].lstrip() except IndexError: pass else: if level_unit == 'cm': level_unit_factor_to_cm = 1 elif level_unit == 'm': level_unit_factor_to_cm = 100 else: level_unit_factor_to_cm = 100 utils.MessagebarAndLog.warning(bar_msg=ru( QCoreApplication.translate( 'LeveloggerImport', '''The unit for level wasn't m or cm, a factor of %s was used. Check the imported data.''' )) % str(level_unit_factor_to_cm)) file_header = rows[data_header_idx] new_header = ['date_time', 'head_cm', 'temp_degc', 'cond_mscm'] filedata.append(new_header) date_colnr = file_header.index('Date') time_colnr = file_header.index('Time') try: level_colnr = file_header.index('LEVEL') except ValueError: level_colnr = None try: temp_colnr = file_header.index('TEMPERATURE') except ValueError: temp_colnr = None try: spec_cond_colnr = file_header.index('spec. conductivity (uS/cm)') except ValueError: try: spec_cond_colnr = file_header.index( 'spec. conductivity (mS/cm)') except ValueError: spec_cond_colnr = None else: spec_cond_factor_to_mScm = 1 else: spec_cond_factor_to_mScm = 0.001 try: first_data_row = rows[data_header_idx + 1] except IndexError: utils.MessagebarAndLog.warning(bar_msg=ru( QCoreApplication.translate( 'LeveloggerImport', '''No data in file %s.''')) % filename) return [], filename, location else: date_str = ' '.join( [first_data_row[date_colnr], first_data_row[time_colnr]]) date_format = date_utils.datestring_to_date(date_str) if date_format is None: utils.MessagebarAndLog.warning(bar_msg=ru( QCoreApplication.translate( 'LeveloggerImport', '''Dateformat in file %s could not be parsed.''')) % filename) return [], filename, location filedata.extend([[ date_utils.long_dateformat( ' '.join([row[date_colnr], row[time_colnr]]), date_format), str( float(row[level_colnr].replace(',', '.')) * level_unit_factor_to_cm) if (utils.to_float_or_none(row[level_colnr]) is not None if level_colnr is not None else None) else None, str(float(row[temp_colnr].replace(',', '.'))) if (utils.to_float_or_none(row[temp_colnr]) if temp_colnr is not None else None) else None, str( float(row[spec_cond_colnr].replace(',', '.')) * spec_cond_factor_to_mScm) if (utils.to_float_or_none(row[spec_cond_colnr]) if spec_cond_colnr is not None else None) else None ] for row in rows[data_header_idx + 1:] if all([ isinstance(utils.to_float_or_none(row[level_colnr]), float ) if skip_rows_without_water_level else True, date_utils.datestring_to_date( ' '.join([row[date_colnr], row[time_colnr]]), df=date_format ) >= begindate if begindate is not None else True, date_utils.datestring_to_date( ' '.join([row[date_colnr], row[time_colnr]]), df=date_format ) <= enddate if enddate is not None else True ])]) filedata = [row for row in filedata if any(row[1:])] return filedata, filename, location
def factor(self, value): as_float = utils.to_float_or_none(value) if isinstance(as_float, float): self._factor.setText(str(as_float))
def parse_hobologger_file(path, charset, skip_rows_without_water_level=False, begindate=None, enddate=None, tz_converter=None): """ Parses a HOBO temperature logger csv file into a string :param path: The file name :param charset: :param skip_rows_without_water_level: :param begindate: :param enddate: :param tz_converter: A TzConverter object. :return: """ filedata = [] location = None filename = os.path.basename(path) if begindate is not None: begindate = date_utils.datestring_to_date(begindate) if enddate is not None: enddate = date_utils.datestring_to_date(enddate) with open(path, 'rt', encoding=str(charset)) as f: rows_unsplit = [row.lstrip().rstrip('\n').rstrip('\r') for row in f] csvreader = csv.reader(rows_unsplit, delimiter=',', quotechar='"') rows = [ru(row, keep_containers=True) for row in csvreader] try: data_header_idx = [rownr for rownr, row in enumerate(rows) if 'Date Time' in '_'.join(row)][0] except IndexError: utils.MessagebarAndLog.warning(bar_msg=ru(QCoreApplication.translate('Hobologger import', '''File %s could not be parsed.'''))%filename) return [], filename, location date_colnr = [idx for idx, col in enumerate(rows[1]) if 'Date Time' in col] if not date_colnr: raise Exception(ru(QCoreApplication.translate('Hobologger import', 'Date Time column not found!'))) else: date_colnr = date_colnr[0] if tz_converter: tz_string = get_tz_string(rows[1][date_colnr]) if tz_string is None: utils.MessagebarAndLog.warning( bar_msg=ru(QCoreApplication.translate('Hobologger import', 'Timezone not found in %s')) % filename) tz_converter.source_tz = tz_string temp_colnr = [idx for idx, col in enumerate(rows[1]) if 'Temp, °C' in col] if not temp_colnr: raise Exception(ru(QCoreApplication.translate('Hobologger import', 'Temperature column not found!'))) else: temp_colnr = temp_colnr[0] match = re.search('LBL: ([A-Za-z0-9_\-]+)', rows[1][temp_colnr]) if not match: location = filename else: location = match.group(1) new_header = ['date_time', 'head_cm', 'temp_degc', 'cond_mscm'] filedata.append(new_header) try: first_data_row = rows[data_header_idx + 1] except IndexError: utils.MessagebarAndLog.warning(bar_msg=ru(QCoreApplication.translate('HobologgerImport', '''No data in file %s.'''))%filename) return [], filename, location else: dt = first_data_row[date_colnr] date_format = date_utils.find_date_format(dt, suppress_error_msg=True) if date_format is None: dt = first_data_row[date_colnr][:-2].rstrip() date_format = date_utils.find_date_format(dt) if date_format is None: utils.MessagebarAndLog.warning(bar_msg=ru(QCoreApplication.translate('HobologgerImport', '''Dateformat in file %s could not be parsed.''')) % filename) return [], filename, location filedata.extend([[date_utils.long_dateformat(fix_date(row[date_colnr], filename, tz_converter)), '', str(float(row[temp_colnr].replace(',', '.'))) if ( utils.to_float_or_none(row[temp_colnr]) if temp_colnr is not None else None) else '', ''] for row in rows[data_header_idx + 1:] if all([fix_date(row[date_colnr], filename, tz_converter) >= begindate if begindate is not None else True, fix_date(row[date_colnr], filename, tz_converter) <= enddate if enddate is not None else True])]) filedata = [row for row in filedata if any(row[1:])] return filedata, filename, location
def parse_hobologger_file(path, charset, skip_rows_without_water_level=False, begindate=None, enddate=None, tz_converter=None): """ Parses a HOBO temperature logger csv file into a string :param path: The file name :param charset: :param skip_rows_without_water_level: :param begindate: :param enddate: :param tz_converter: A TzConverter object. :return: """ filedata = [] location = None filename = os.path.basename(path) if begindate is not None: begindate = date_utils.datestring_to_date(begindate) if enddate is not None: enddate = date_utils.datestring_to_date(enddate) with open(path, 'rt', encoding=str(charset)) as f: rows_unsplit = [row.lstrip().rstrip('\n').rstrip('\r') for row in f] csvreader = csv.reader(rows_unsplit, delimiter=',', quotechar='"') rows = [ru(row, keep_containers=True) for row in csvreader] try: data_header_idx = [rownr for rownr, row in enumerate(rows) if 'Date Time' in '_'.join(row)][0] except IndexError: utils.MessagebarAndLog.warning(bar_msg=ru(QCoreApplication.translate('Hobologger import', '''File %s could not be parsed.'''))%filename) return [], filename, location date_colnr = [idx for idx, col in enumerate(rows[1]) if 'Date Time' in col] if not date_colnr: raise Exception(ru(QCoreApplication.translate('Hobologger import', 'Date Time column not found!'))) else: date_colnr = date_colnr[0] if tz_converter: tz_string = get_tz_string(rows[1][date_colnr]) if tz_string is None: utils.MessagebarAndLog.warning( bar_msg=ru(QCoreApplication.translate('Hobologger import', 'Timezone not found in %s')) % filename) tz_converter.source_tz = tz_string temp_colnr = [idx for idx, col in enumerate(rows[1]) if 'Temp, °C' in col] if not temp_colnr: raise Exception(ru(QCoreApplication.translate('Hobologger import', 'Temperature column not found!'))) else: temp_colnr = temp_colnr[0] match = re.search('LBL: ([A-Za-z0-9_\-]+)', rows[1][temp_colnr]) if not match: location = filename else: location = match.group(1) new_header = ['date_time', 'head_cm', 'temp_degc', 'cond_mscm'] filedata.append(new_header) try: first_data_row = rows[data_header_idx + 1] except IndexError: utils.MessagebarAndLog.warning(bar_msg=ru(QCoreApplication.translate('HobologgerImport', '''No data in file %s.'''))%filename) return [], filename, location else: dt = first_data_row[date_colnr] date_format = date_utils.find_date_format(dt, suppress_error_msg=True) if date_format is None: dt = first_data_row[date_colnr][:-2].rstrip() date_format = date_utils.find_date_format(dt) if date_format is None: utils.MessagebarAndLog.warning(bar_msg=ru(QCoreApplication.translate('HobologgerImport', '''Dateformat in file %s could not be parsed.''')) % filename) return [], filename, location filedata.extend([[date_utils.long_dateformat(fix_date(row[date_colnr], filename, tz_converter)), '', str(float(row[temp_colnr].replace(',', '.'))) if ( utils.to_float_or_none(row[temp_colnr]) if temp_colnr is not None else None) else '', ''] for row in rows[data_header_idx + 1:] if all([fix_date(row[date_colnr], filename, tz_converter) >= begindate if begindate is not None else True, fix_date(row[date_colnr], filename, tz_converter) <= enddate if enddate is not None else True])]) filedata = [row for row in filedata if any(row[1:])] return filedata, filename, location
def parse_levelogger_file(path, charset, skip_rows_without_water_level=False, begindate=None, enddate=None): """ Parses a levelogger csv file into a string :param path: The file name :param existing_obsids: A list or tuple with the obsids that exist in the db. :param ask_for_names: (True/False) True to ask for location name for every location. False to only ask if the location is not found in existing_obsids. :return: A string representing a table file. Including '\n' for newlines. Assumptions and limitations: * The Location attribute is used as location and added as a column. * Values containing ',' is replaced with '.' * Rows with missing "Water head[cm]"-data is skipped. """ #These can be set to paritally import files. #begindate = datetime.strptime('2016-06-08 20:00:00','%Y-%m-%d %H:%M:%S') #enddate = datetime.strptime('2016-06-08 19:00:00','%Y-%m-%d %H:%M:%S') #It should be possible to import all cols that exists in the translation dict filedata = [] location = None level_unit_factor_to_cm = 100 spec_cond_factor_to_mScm = 0.001 filename = os.path.basename(path) if begindate is not None: begindate = date_utils.datestring_to_date(begindate) if enddate is not None: enddate = date_utils.datestring_to_date(enddate) with io.open(path, 'rt', encoding=str(charset)) as f: rows_unsplit = [row.lstrip().rstrip('\n').rstrip('\r') for row in f] try: data_header_idx = [rownr for rownr, row in enumerate(rows_unsplit) if row.startswith('Date')][0] except IndexError: utils.MessagebarAndLog.warning(bar_msg=ru(QCoreApplication.translate('LeveloggerImport', '''File %s could not be parsed.'''))%filename) return [], filename, location delimiter = utils.get_delimiter_from_file_rows(rows_unsplit[data_header_idx:], filename=filename, delimiters=[';', ','], num_fields=None) if delimiter is None: return [], filename, location rows = [row.split(';') for row in rows_unsplit] col1 = [row[0] for row in rows] try: location_idx = col1.index('Location:') except ValueError: pass else: location = col1[location_idx + 1] try: level_unit_idx = col1.index('LEVEL') except ValueError: pass else: try: level_unit = col1[level_unit_idx + 1].split(':')[1].lstrip() except IndexError: pass else: if level_unit == 'cm': level_unit_factor_to_cm = 1 elif level_unit == 'm': level_unit_factor_to_cm = 100 else: level_unit_factor_to_cm = 100 utils.MessagebarAndLog.warning(bar_msg=ru(QCoreApplication.translate('LeveloggerImport', '''The unit for level wasn't m or cm, a factor of %s was used. Check the imported data.'''))%str(level_unit_factor_to_cm)) file_header = rows[data_header_idx] new_header = ['date_time', 'head_cm', 'temp_degc', 'cond_mscm'] filedata.append(new_header) date_colnr = file_header.index('Date') time_colnr = file_header.index('Time') try: level_colnr = file_header.index('LEVEL') except ValueError: level_colnr = None try: temp_colnr = file_header.index('TEMPERATURE') except ValueError: temp_colnr = None try: spec_cond_colnr = file_header.index('spec. conductivity (uS/cm)') except ValueError: try: spec_cond_colnr = file_header.index('spec. conductivity (mS/cm)') except ValueError: spec_cond_colnr = None else: spec_cond_factor_to_mScm = 1 else: spec_cond_factor_to_mScm = 0.001 try: first_data_row = rows[data_header_idx + 1] except IndexError: utils.MessagebarAndLog.warning(bar_msg=ru(QCoreApplication.translate('LeveloggerImport', '''No data in file %s.'''))%filename) return [], filename, location else: date_str = ' '.join([first_data_row[date_colnr], first_data_row[time_colnr]]) date_format = date_utils.datestring_to_date(date_str) if date_format is None: utils.MessagebarAndLog.warning(bar_msg=ru(QCoreApplication.translate('LeveloggerImport', '''Dateformat in file %s could not be parsed.''')) % filename) return [], filename, location filedata.extend([[date_utils.long_dateformat(' '.join([row[date_colnr], row[time_colnr]]), date_format), str(float(row[level_colnr].replace(',', '.')) * level_unit_factor_to_cm) if (utils.to_float_or_none(row[level_colnr]) is not None if level_colnr is not None else None) else None, str(float(row[temp_colnr].replace(',', '.'))) if (utils.to_float_or_none(row[temp_colnr]) if temp_colnr is not None else None) else None, str(float(row[spec_cond_colnr].replace(',', '.')) * spec_cond_factor_to_mScm) if (utils.to_float_or_none(row[spec_cond_colnr]) if spec_cond_colnr is not None else None) else None] for row in rows[data_header_idx + 1:] if all([isinstance(utils.to_float_or_none(row[level_colnr]), float) if skip_rows_without_water_level else True, date_utils.datestring_to_date(' '.join([row[date_colnr], row[time_colnr]]), df=date_format) >= begindate if begindate is not None else True, date_utils.datestring_to_date(' '.join([row[date_colnr], row[time_colnr]]), df=date_format) <= enddate if enddate is not None else True])]) filedata = [row for row in filedata if any(row[1:])] return filedata, filename, location