def factor(self):
value = self._factor.text()
as_float = utils.to_float_or_none(value)
if isinstance(as_float, float):
return as_float
else:
return None
def factor(self):
value = self._factor.text()
as_float = utils.to_float_or_none(value)
if isinstance(as_float, float):
return as_float
else:
return None
def multiply_by_factor(file_data, columns_factors):
"""
Multiplies all values in the file data with a given factor for each column
:param file_data: a list of lists like [['obsid', 'date_time', 'reading'], ['obs1', '2017-04-12 11:03', '123,456']]
:param table_columns_factors: a dict like {'reading': 10}
:return: file_data where the columns have been multiplied by the factor.
"""
file_data = [[str(float(col) * columns_factors[file_data[0][colnr]]) if
(file_data[0][colnr] in columns_factors and utils.to_float_or_none(col) is not None)
else col for colnr, col in enumerate(row)]
if rownr > 0 else row for rownr, row in enumerate(file_data)]
return file_data
def multiply_by_factor(file_data, columns_factors):
"""
Multiplies all values in the file data with a given factor for each column
:param file_data: a list of lists like [['obsid', 'date_time', 'reading'], ['obs1', '2017-04-12 11:03', '123,456']]
:param table_columns_factors: a dict like {'reading': 10}
:return: file_data where the columns have been multiplied by the factor.
"""
file_data = [[str(float(col) * columns_factors[file_data[0][colnr]]) if
(file_data[0][colnr] in columns_factors and utils.to_float_or_none(col) is not None)
else col for colnr, col in enumerate(row)]
if rownr > 0 else row for rownr, row in enumerate(file_data)]
return file_data
def factor(self, value):
as_float = utils.to_float_or_none(value)
if isinstance(as_float, float):
self._factor.setText(str(as_float))
def parse_levelogger_file(path,
charset,
skip_rows_without_water_level=False,
begindate=None,
enddate=None):
""" Parses a levelogger csv file into a string
:param path: The file name
:param existing_obsids: A list or tuple with the obsids that exist in the db.
:param ask_for_names: (True/False) True to ask for location name for every location. False to only ask if the location is not found in existing_obsids.
:return: A string representing a table file. Including '\n' for newlines.
Assumptions and limitations:
* The Location attribute is used as location and added as a column.
* Values containing ',' is replaced with '.'
* Rows with missing "Water head[cm]"-data is skipped.
"""
#These can be set to paritally import files.
#begindate = datetime.strptime('2016-06-08 20:00:00','%Y-%m-%d %H:%M:%S')
#enddate = datetime.strptime('2016-06-08 19:00:00','%Y-%m-%d %H:%M:%S')
#It should be possible to import all cols that exists in the translation dict
filedata = []
location = None
level_unit_factor_to_cm = 100
spec_cond_factor_to_mScm = 0.001
filename = os.path.basename(path)
if begindate is not None:
begindate = date_utils.datestring_to_date(begindate)
if enddate is not None:
enddate = date_utils.datestring_to_date(enddate)
with io.open(path, 'rt', encoding=str(charset)) as f:
rows_unsplit = [
row.lstrip().rstrip('\n').rstrip('\r') for row in f
]
try:
data_header_idx = [
rownr for rownr, row in enumerate(rows_unsplit)
if row.startswith('Date')
][0]
except IndexError:
utils.MessagebarAndLog.warning(bar_msg=ru(
QCoreApplication.translate(
'LeveloggerImport', '''File %s could not be parsed.''')) %
filename)
return [], filename, location
delimiter = utils.get_delimiter_from_file_rows(
rows_unsplit[data_header_idx:],
filename=filename,
delimiters=[';', ','],
num_fields=None)
if delimiter is None:
return [], filename, location
rows = [row.split(';') for row in rows_unsplit]
lens = set([len(row) for row in rows[data_header_idx:]])
if len(lens) != 1 or list(lens)[0] == 1:
# Assume that the delimiter was not ';'
rows = [row.split(',') for row in rows_unsplit]
col1 = [row[0] for row in rows]
try:
location_idx = col1.index('Location:')
except ValueError:
pass
else:
location = col1[location_idx + 1]
try:
level_unit_idx = col1.index('LEVEL')
except ValueError:
pass
else:
try:
level_unit = col1[level_unit_idx + 1].split(':')[1].lstrip()
except IndexError:
pass
else:
if level_unit == 'cm':
level_unit_factor_to_cm = 1
elif level_unit == 'm':
level_unit_factor_to_cm = 100
else:
level_unit_factor_to_cm = 100
utils.MessagebarAndLog.warning(bar_msg=ru(
QCoreApplication.translate(
'LeveloggerImport',
'''The unit for level wasn't m or cm, a factor of %s was used. Check the imported data.'''
)) % str(level_unit_factor_to_cm))
file_header = rows[data_header_idx]
new_header = ['date_time', 'head_cm', 'temp_degc', 'cond_mscm']
filedata.append(new_header)
date_colnr = file_header.index('Date')
time_colnr = file_header.index('Time')
try:
level_colnr = file_header.index('LEVEL')
except ValueError:
level_colnr = None
try:
temp_colnr = file_header.index('TEMPERATURE')
except ValueError:
temp_colnr = None
try:
spec_cond_colnr = file_header.index('spec. conductivity (uS/cm)')
except ValueError:
try:
spec_cond_colnr = file_header.index(
'spec. conductivity (mS/cm)')
except ValueError:
spec_cond_colnr = None
else:
spec_cond_factor_to_mScm = 1
else:
spec_cond_factor_to_mScm = 0.001
try:
first_data_row = rows[data_header_idx + 1]
except IndexError:
utils.MessagebarAndLog.warning(bar_msg=ru(
QCoreApplication.translate(
'LeveloggerImport', '''No data in file %s.''')) % filename)
return [], filename, location
else:
date_str = ' '.join(
[first_data_row[date_colnr], first_data_row[time_colnr]])
date_format = date_utils.datestring_to_date(date_str)
if date_format is None:
utils.MessagebarAndLog.warning(bar_msg=ru(
QCoreApplication.translate(
'LeveloggerImport',
'''Dateformat in file %s could not be parsed.''')) %
filename)
return [], filename, location
filedata.extend([[
date_utils.long_dateformat(
' '.join([row[date_colnr], row[time_colnr]]), date_format),
str(
float(row[level_colnr].replace(',', '.')) *
level_unit_factor_to_cm) if
(utils.to_float_or_none(row[level_colnr]) is not None
if level_colnr is not None else None) else None,
str(float(row[temp_colnr].replace(',', '.'))) if
(utils.to_float_or_none(row[temp_colnr])
if temp_colnr is not None else None) else None,
str(
float(row[spec_cond_colnr].replace(',', '.')) *
spec_cond_factor_to_mScm) if
(utils.to_float_or_none(row[spec_cond_colnr])
if spec_cond_colnr is not None else None) else None
] for row in rows[data_header_idx + 1:] if all([
isinstance(utils.to_float_or_none(row[level_colnr]), float
) if skip_rows_without_water_level else True,
date_utils.datestring_to_date(
' '.join([row[date_colnr], row[time_colnr]]), df=date_format
) >= begindate if begindate is not None else True,
date_utils.datestring_to_date(
' '.join([row[date_colnr], row[time_colnr]]), df=date_format
) <= enddate if enddate is not None else True
])])
filedata = [row for row in filedata if any(row[1:])]
return filedata, filename, location
def factor(self, value):
as_float = utils.to_float_or_none(value)
if isinstance(as_float, float):
self._factor.setText(str(as_float))
def parse_hobologger_file(path, charset, skip_rows_without_water_level=False, begindate=None, enddate=None, tz_converter=None):
""" Parses a HOBO temperature logger csv file into a string
:param path: The file name
:param charset:
:param skip_rows_without_water_level:
:param begindate:
:param enddate:
:param tz_converter: A TzConverter object.
:return:
"""
filedata = []
location = None
filename = os.path.basename(path)
if begindate is not None:
begindate = date_utils.datestring_to_date(begindate)
if enddate is not None:
enddate = date_utils.datestring_to_date(enddate)
with open(path, 'rt', encoding=str(charset)) as f:
rows_unsplit = [row.lstrip().rstrip('\n').rstrip('\r') for row in f]
csvreader = csv.reader(rows_unsplit, delimiter=',', quotechar='"')
rows = [ru(row, keep_containers=True) for row in csvreader]
try:
data_header_idx = [rownr for rownr, row in enumerate(rows) if 'Date Time' in '_'.join(row)][0]
except IndexError:
utils.MessagebarAndLog.warning(bar_msg=ru(QCoreApplication.translate('Hobologger import',
'''File %s could not be parsed.'''))%filename)
return [], filename, location
date_colnr = [idx for idx, col in enumerate(rows[1]) if 'Date Time' in col]
if not date_colnr:
raise Exception(ru(QCoreApplication.translate('Hobologger import', 'Date Time column not found!')))
else:
date_colnr = date_colnr[0]
if tz_converter:
tz_string = get_tz_string(rows[1][date_colnr])
if tz_string is None:
utils.MessagebarAndLog.warning(
bar_msg=ru(QCoreApplication.translate('Hobologger import', 'Timezone not found in %s')) % filename)
tz_converter.source_tz = tz_string
temp_colnr = [idx for idx, col in enumerate(rows[1]) if 'Temp, °C' in col]
if not temp_colnr:
raise Exception(ru(QCoreApplication.translate('Hobologger import', 'Temperature column not found!')))
else:
temp_colnr = temp_colnr[0]
match = re.search('LBL: ([A-Za-z0-9_\-]+)', rows[1][temp_colnr])
if not match:
location = filename
else:
location = match.group(1)
new_header = ['date_time', 'head_cm', 'temp_degc', 'cond_mscm']
filedata.append(new_header)
try:
first_data_row = rows[data_header_idx + 1]
except IndexError:
utils.MessagebarAndLog.warning(bar_msg=ru(QCoreApplication.translate('HobologgerImport',
'''No data in file %s.'''))%filename)
return [], filename, location
else:
dt = first_data_row[date_colnr]
date_format = date_utils.find_date_format(dt, suppress_error_msg=True)
if date_format is None:
dt = first_data_row[date_colnr][:-2].rstrip()
date_format = date_utils.find_date_format(dt)
if date_format is None:
utils.MessagebarAndLog.warning(bar_msg=ru(QCoreApplication.translate('HobologgerImport',
'''Dateformat in file %s could not be parsed.''')) % filename)
return [], filename, location
filedata.extend([[date_utils.long_dateformat(fix_date(row[date_colnr], filename, tz_converter)),
'',
str(float(row[temp_colnr].replace(',', '.'))) if (
utils.to_float_or_none(row[temp_colnr]) if temp_colnr is not None else None) else '',
'']
for row in rows[data_header_idx + 1:]
if all([fix_date(row[date_colnr], filename, tz_converter) >= begindate if begindate is not None else True,
fix_date(row[date_colnr], filename, tz_converter) <= enddate if enddate is not None else True])])
filedata = [row for row in filedata if any(row[1:])]
return filedata, filename, location
def parse_hobologger_file(path, charset, skip_rows_without_water_level=False, begindate=None, enddate=None, tz_converter=None):
""" Parses a HOBO temperature logger csv file into a string
:param path: The file name
:param charset:
:param skip_rows_without_water_level:
:param begindate:
:param enddate:
:param tz_converter: A TzConverter object.
:return:
"""
filedata = []
location = None
filename = os.path.basename(path)
if begindate is not None:
begindate = date_utils.datestring_to_date(begindate)
if enddate is not None:
enddate = date_utils.datestring_to_date(enddate)
with open(path, 'rt', encoding=str(charset)) as f:
rows_unsplit = [row.lstrip().rstrip('\n').rstrip('\r') for row in f]
csvreader = csv.reader(rows_unsplit, delimiter=',', quotechar='"')
rows = [ru(row, keep_containers=True) for row in csvreader]
try:
data_header_idx = [rownr for rownr, row in enumerate(rows) if 'Date Time' in '_'.join(row)][0]
except IndexError:
utils.MessagebarAndLog.warning(bar_msg=ru(QCoreApplication.translate('Hobologger import',
'''File %s could not be parsed.'''))%filename)
return [], filename, location
date_colnr = [idx for idx, col in enumerate(rows[1]) if 'Date Time' in col]
if not date_colnr:
raise Exception(ru(QCoreApplication.translate('Hobologger import', 'Date Time column not found!')))
else:
date_colnr = date_colnr[0]
if tz_converter:
tz_string = get_tz_string(rows[1][date_colnr])
if tz_string is None:
utils.MessagebarAndLog.warning(
bar_msg=ru(QCoreApplication.translate('Hobologger import', 'Timezone not found in %s')) % filename)
tz_converter.source_tz = tz_string
temp_colnr = [idx for idx, col in enumerate(rows[1]) if 'Temp, °C' in col]
if not temp_colnr:
raise Exception(ru(QCoreApplication.translate('Hobologger import', 'Temperature column not found!')))
else:
temp_colnr = temp_colnr[0]
match = re.search('LBL: ([A-Za-z0-9_\-]+)', rows[1][temp_colnr])
if not match:
location = filename
else:
location = match.group(1)
new_header = ['date_time', 'head_cm', 'temp_degc', 'cond_mscm']
filedata.append(new_header)
try:
first_data_row = rows[data_header_idx + 1]
except IndexError:
utils.MessagebarAndLog.warning(bar_msg=ru(QCoreApplication.translate('HobologgerImport',
'''No data in file %s.'''))%filename)
return [], filename, location
else:
dt = first_data_row[date_colnr]
date_format = date_utils.find_date_format(dt, suppress_error_msg=True)
if date_format is None:
dt = first_data_row[date_colnr][:-2].rstrip()
date_format = date_utils.find_date_format(dt)
if date_format is None:
utils.MessagebarAndLog.warning(bar_msg=ru(QCoreApplication.translate('HobologgerImport',
'''Dateformat in file %s could not be parsed.''')) % filename)
return [], filename, location
filedata.extend([[date_utils.long_dateformat(fix_date(row[date_colnr], filename, tz_converter)),
'',
str(float(row[temp_colnr].replace(',', '.'))) if (
utils.to_float_or_none(row[temp_colnr]) if temp_colnr is not None else None) else '',
'']
for row in rows[data_header_idx + 1:]
if all([fix_date(row[date_colnr], filename, tz_converter) >= begindate if begindate is not None else True,
fix_date(row[date_colnr], filename, tz_converter) <= enddate if enddate is not None else True])])
filedata = [row for row in filedata if any(row[1:])]
return filedata, filename, location
def parse_levelogger_file(path, charset, skip_rows_without_water_level=False, begindate=None, enddate=None):
""" Parses a levelogger csv file into a string
:param path: The file name
:param existing_obsids: A list or tuple with the obsids that exist in the db.
:param ask_for_names: (True/False) True to ask for location name for every location. False to only ask if the location is not found in existing_obsids.
:return: A string representing a table file. Including '\n' for newlines.
Assumptions and limitations:
* The Location attribute is used as location and added as a column.
* Values containing ',' is replaced with '.'
* Rows with missing "Water head[cm]"-data is skipped.
"""
#These can be set to paritally import files.
#begindate = datetime.strptime('2016-06-08 20:00:00','%Y-%m-%d %H:%M:%S')
#enddate = datetime.strptime('2016-06-08 19:00:00','%Y-%m-%d %H:%M:%S')
#It should be possible to import all cols that exists in the translation dict
filedata = []
location = None
level_unit_factor_to_cm = 100
spec_cond_factor_to_mScm = 0.001
filename = os.path.basename(path)
if begindate is not None:
begindate = date_utils.datestring_to_date(begindate)
if enddate is not None:
enddate = date_utils.datestring_to_date(enddate)
with io.open(path, 'rt', encoding=str(charset)) as f:
rows_unsplit = [row.lstrip().rstrip('\n').rstrip('\r') for row in f]
try:
data_header_idx = [rownr for rownr, row in enumerate(rows_unsplit) if row.startswith('Date')][0]
except IndexError:
utils.MessagebarAndLog.warning(bar_msg=ru(QCoreApplication.translate('LeveloggerImport',
'''File %s could not be parsed.'''))%filename)
return [], filename, location
delimiter = utils.get_delimiter_from_file_rows(rows_unsplit[data_header_idx:], filename=filename, delimiters=[';', ','], num_fields=None)
if delimiter is None:
return [], filename, location
rows = [row.split(';') for row in rows_unsplit]
col1 = [row[0] for row in rows]
try:
location_idx = col1.index('Location:')
except ValueError:
pass
else:
location = col1[location_idx + 1]
try:
level_unit_idx = col1.index('LEVEL')
except ValueError:
pass
else:
try:
level_unit = col1[level_unit_idx + 1].split(':')[1].lstrip()
except IndexError:
pass
else:
if level_unit == 'cm':
level_unit_factor_to_cm = 1
elif level_unit == 'm':
level_unit_factor_to_cm = 100
else:
level_unit_factor_to_cm = 100
utils.MessagebarAndLog.warning(bar_msg=ru(QCoreApplication.translate('LeveloggerImport', '''The unit for level wasn't m or cm, a factor of %s was used. Check the imported data.'''))%str(level_unit_factor_to_cm))
file_header = rows[data_header_idx]
new_header = ['date_time', 'head_cm', 'temp_degc', 'cond_mscm']
filedata.append(new_header)
date_colnr = file_header.index('Date')
time_colnr = file_header.index('Time')
try:
level_colnr = file_header.index('LEVEL')
except ValueError:
level_colnr = None
try:
temp_colnr = file_header.index('TEMPERATURE')
except ValueError:
temp_colnr = None
try:
spec_cond_colnr = file_header.index('spec. conductivity (uS/cm)')
except ValueError:
try:
spec_cond_colnr = file_header.index('spec. conductivity (mS/cm)')
except ValueError:
spec_cond_colnr = None
else:
spec_cond_factor_to_mScm = 1
else:
spec_cond_factor_to_mScm = 0.001
try:
first_data_row = rows[data_header_idx + 1]
except IndexError:
utils.MessagebarAndLog.warning(bar_msg=ru(QCoreApplication.translate('LeveloggerImport',
'''No data in file %s.'''))%filename)
return [], filename, location
else:
date_str = ' '.join([first_data_row[date_colnr], first_data_row[time_colnr]])
date_format = date_utils.datestring_to_date(date_str)
if date_format is None:
utils.MessagebarAndLog.warning(bar_msg=ru(QCoreApplication.translate('LeveloggerImport',
'''Dateformat in file %s could not be parsed.''')) % filename)
return [], filename, location
filedata.extend([[date_utils.long_dateformat(' '.join([row[date_colnr], row[time_colnr]]), date_format),
str(float(row[level_colnr].replace(',', '.')) * level_unit_factor_to_cm) if (utils.to_float_or_none(row[level_colnr]) is not None if level_colnr is not None else None) else None,
str(float(row[temp_colnr].replace(',', '.'))) if (utils.to_float_or_none(row[temp_colnr]) if temp_colnr is not None else None) else None,
str(float(row[spec_cond_colnr].replace(',', '.')) * spec_cond_factor_to_mScm) if (utils.to_float_or_none(row[spec_cond_colnr]) if spec_cond_colnr is not None else None) else None]
for row in rows[data_header_idx + 1:]
if all([isinstance(utils.to_float_or_none(row[level_colnr]), float) if skip_rows_without_water_level else True,
date_utils.datestring_to_date(' '.join([row[date_colnr], row[time_colnr]]), df=date_format) >= begindate if begindate is not None else True,
date_utils.datestring_to_date(' '.join([row[date_colnr], row[time_colnr]]), df=date_format) <= enddate if enddate is not None else True])])
filedata = [row for row in filedata if any(row[1:])]
return filedata, filename, location