def generic_testing(directory):
end = int(
DateTimeUtil.local_time_str_to_utc('2019/04/29 18:00:00').timestamp())
# David
start = int(
DateTimeUtil.local_time_str_to_utc('2019/04/03 18:00:00').timestamp())
testing_set('measured_david', start, end,
'{0}/gt_david.csv'.format(directory))
# Martin
start = int(
DateTimeUtil.local_time_str_to_utc('2019/04/01 18:00:00').timestamp())
testing_set('measured_martin', start, end,
'{0}/gt_martin.csv'.format(directory))
# Peto , februar, marec, april
start = int(
DateTimeUtil.local_time_str_to_utc('2019/02/04 18:00:00').timestamp())
testing_set('measured_filtered_peto', start, end,
'{0}/gt_peto.csv'.format(directory))
# Klarka
start = int(
DateTimeUtil.local_time_str_to_utc('2018/12/18 18:00:00').timestamp())
testing_set('measured_klarka', start, end,
'{0}/gt_klarka.csv'.format(directory))
def generate_file(con, start_shift, end_shift, output_file, enable_regression):
logging.info('start: ' + output_file)
graphs = Graph("./../../src/graph")
# stiahnutie dat
storage = Storage('examples/events_peto.json', 0, 'measured_filtered_peto')
d = storage.load_data(con, start_shift, end_shift, 'co2_in_ppm')
logging.info('downloaded events: %d' % len(d))
# aplikovanie filtrov na eventy
filtered = FilterUtil.only_valid_events(d)
# for travis
if ConnectionUtil.is_testable_system():
filtered = filtered[:ConnectionUtil.MAX_TESTABLE_EVENTS]
logging.info('events after applying the filter: %d' % len(filtered))
# spocitanie regresie
if enable_regression:
filtered = compute_regression(filtered)
logging.info('start generating graphs')
gr = []
for event in filtered:
t = DateTimeUtil.utc_timestamp_to_str(event['e_start']['timestamp'], '%d.%m. %H:%M:%S')
t += ' - '
t += DateTimeUtil.utc_timestamp_to_str(event['e_end']['timestamp'], '%H:%M:%S')
if enable_regression:
gg = [
Graph.db_to_simple_graph(event, 'co2_in_ppm', 'green', 'CO2', 50),
Graph.db_to_simple_graph(event, 'co2_in_ppm_exp', 'red', 'SimpleExpRegression', 50),
Graph.db_to_simple_graph(event, 'co2_in_ppm_exp2', 'orange', 'ExpRegressionWithDelay', 50),
]
else:
gg = [
Graph.db_to_simple_graph(event, 'co2_in_ppm', 'green', 'CO2', 50),
]
g = {
'title': t,
'graphs': gg
}
gr.append(g)
graphs.gen(gr, output_file + '.html', 0, 0)
logging.info('end generating graphs')
logging.info('end')
def row(self, column_name, time):
if column_name not in self.cache:
self.cache[column_name] = {}
res = Storage.select_interval(self.con,
self.start,
self.end,
column_name,
self.table_name,
without_none_value=False)
actual_timestamp = self.start
for row in res:
if row is None:
self.cache[column_name][actual_timestamp] = None
else:
self.cache[column_name][actual_timestamp] = float(row)
actual_timestamp += 1
if time in self.cache[column_name]:
value = self.cache[column_name][time]
else:
value = super(CachedRowWithIntervalSelector,
self).row(column_name, time)
if value is None:
t = DateTimeUtil.utc_timestamp_to_str(time, '%Y/%m/%d %H:%M:%S')
raise ValueError('empty value at %s' % t)
return value
def window_no_event_value(values: tuple, precision: int):
outer_co2_ppm = 435
p = precision
temperature_diff = values[4] - values[6]
humidity_rh_diff = values[7] - values[13]
humidity_abs_diff = values[9] - values[14]
humidity_spec_diff = values[11] - values[15]
return OrderedDict([
('datetime', DateTimeUtil.utc_timestamp_to_str(values[0], '%Y-%m-%d %H:%M:%S')),
('event', 'nothing'),
('co2_in_ppm', round(values[16], p)),
('co2_out_ppm', round(outer_co2_ppm, p)),
('temperature_in_celsius', round(values[4], p)),
('temperature_out_celsius', round(values[6], p)),
('humidity_in_relative_percent', round(values[7], p)),
('humidity_in_absolute_g_m3', round(values[9], p)),
('humidity_in_specific_g_kg', round(values[11], p)),
('humidity_out_relative_percent', round(values[13], p)),
('humidity_out_absolute_g_m3', round(values[14], p)),
('humidity_out_specific_g_kg', round(values[15], p)),
('pressure_in_hpa', round(values[4], p)),
('temperature_celsius_difference', round(temperature_diff, p)),
('humidity_relative_percent_difference', round(humidity_rh_diff, p)),
('humidity_absolute_g_m3_difference', round(humidity_abs_diff, p)),
('humidity_specific_g_kg_difference', round(humidity_spec_diff, p)),
])
def row(self, column_name, time):
res = Storage.one_row(self.con, self.table_name, column_name, time)
if res is None or res[0] is None:
t = DateTimeUtil.utc_timestamp_to_str(time, '%Y/%m/%d %H:%M:%S')
raise ValueError('empty value at %s' % t)
return float(res[0])
def additional_training_set(con,
table_name,
no_event_records,
func,
row_selector,
interval_selector,
print_each=10):
"""Supplementary generation of training data based on given time points.
:param con:
:param table_name: table name
:param no_event_records: list of pairs for attribute generation
:param func:
:param row_selector:
:param interval_selector:
:return:
"""
attrs = []
for k in range(0, len(no_event_records)):
row = no_event_records[k]
if k % print_each == 0:
logging.debug('{0}/{1} events'.format(k,
len(no_event_records)))
if row[0] == '':
logging.warning('empty row in additional sets')
continue
start = int(DateTimeUtil.local_time_str_to_utc(row[0]).timestamp())
try:
data1 = func(con, table_name, start, row_selector,
interval_selector)
time = DateTimeUtil.utc_timestamp_to_str(
start, '%Y/%m/%d %H:%M:%S')
data1.insert(0, ('datetime', time))
data1.insert(1, ('event', row[1]))
attrs.append(OrderedDict(data1))
except Exception as e:
logging.error(str(e))
continue
return attrs
def row(self, column_name, time):
if column_name not in self.cache:
self.cache[column_name] = {}
if time in self.cache[column_name]:
value = self.cache[column_name][time]
else:
start = time - self.half_window_size
end = time + self.half_window_size
res = Storage.select_interval(self.con, start, end, column_name, self.table_name,
without_none_value=False)
error = False
if res is None or None in res:
error = True
if error:
self.cache[column_name][time] = None
t = DateTimeUtil.utc_timestamp_to_str(time, '%Y/%m/%d %H:%M:%S')
raise ValueError('empty value at %s' % t)
x = []
y = []
for i in range(0, len(res)):
x.append(i)
y.append(res[i])
slope, intercept, _, _, _ = stats.linregress(x, y)
value = intercept + slope * self.half_window_size
self.cache[column_name][time] = value
if value is None:
t = DateTimeUtil.utc_timestamp_to_str(time, '%Y/%m/%d %H:%M:%S')
raise ValueError('empty value at %s' % t)
return value
def value_filter(data):
for i in range(0, len(data)):
row = data[i]
for k in range(0, len(row)):
item = row[k]
key = 'rh_in_percentage'
if key in item and item[key] is not None and float(
item[key]) > 100:
t = DateTimeUtil.utc_timestamp_to_str(
item['measured_time'])
value = float(item[key])
logging.error(
'{0}, {1}: value {2} is out of range, skipped'.format(
t, key, value))
item[key] = None
return data
def row(self, column_name, time):
if column_name not in self.cache:
self.cache[column_name] = {}
value = None
if time in self.cache[column_name]:
value = self.cache[column_name][time]
else:
res = Storage.one_row(self.con, self.table_name, column_name, time)
if res is not None and res[0] is not None:
self.cache[column_name][time] = float(res[0])
value = float(res[0])
if value is None:
t = DateTimeUtil.utc_timestamp_to_str(time, '%Y/%m/%d %H:%M:%S')
raise ValueError('empty value at %s' % t)
return value
def check_timestamp_order(con, table_name):
"""Checks if a given table in database does not contain missing value.
:param con: connection with database
:param table_name: table name
"""
cur = con.cursor()
rows = DBUtil.rows_count(con, table_name)
if rows == 0:
logging.debug('DB is empty')
return
first_inserted_timestamp = DBUtil.first_inserted_values(
con, table_name)[0]
last_inserted_timestamp = DBUtil.last_inserted_values(con,
table_name)[0]
last_checked_timestamp = first_inserted_timestamp
if (last_inserted_timestamp - first_inserted_timestamp + 1) != rows:
raise ValueError('DB contains missing or extra rows')
step = 100000
for i in range(0, rows, step):
sql = 'SELECT measured_time, measured_time_str FROM '
sql += table_name + ' ORDER BY measured_time ASC'
sql += ' LIMIT ' + str(i) + ', ' + str(step)
interval = DateTimeUtil.create_interval_str(
first_inserted_timestamp + i,
first_inserted_timestamp + i + step)
logging.debug('check timestamp between %s' % interval)
cur.execute(sql)
res = cur.fetchall()
for r in res:
if last_checked_timestamp == r[0]:
last_checked_timestamp += 1
else:
raise IndexError('missing or extra row: ' +
str(last_checked_timestamp))
def insert_values(conn, table_name, values, maps, write_each, precision):
for i in range(0, len(values)):
value = values[i]
t = ()
if i % write_each != 0:
continue
for column in DBUtil.measured_values_table_column_names():
if column == PreProcessing.TIME_STRING_ATTR_NAME:
t += (DateTimeUtil.utc_timestamp_to_str(
value[PreProcessing.TIME_ATTR_NAME]), )
continue
if column in maps and value[column] is not None:
t += (round(value[column], precision), )
else:
t += (None, )
DBUtil.insert_value(conn, t, False, table_name)
def window_event_value(measured: dict, value_index: int, timestamp: int, precision: int):
outer_co2_ppm = 435
p = precision
i = value_index
temperature_diff = measured['temperature_in_celsius'][i] - \
measured['temperature_out_celsius'][i]
humidity_rh_diff = measured['rh_in_percentage'][i] - \
measured['rh_out_percentage'][i]
humidity_abs_diff = measured['rh_in_absolute_g_m3'][i] - \
measured['rh_out_absolute_g_m3'][i]
humidity_spec_diff = measured['rh_in_specific_g_kg'][i] - \
measured['rh_out_specific_g_kg'][i]
return OrderedDict([
('datetime', DateTimeUtil.utc_timestamp_to_str(timestamp, '%Y-%m-%d %H:%M:%S')),
('event', ValueUtil.detect_window_action(len(measured['co2_in_ppm']), value_index)),
('co2_in_ppm', round(measured['co2_in_ppm'][i], p)),
('co2_out_ppm', round(outer_co2_ppm, p)),
('temperature_in_celsius', round(measured['temperature_in_celsius'][i], p)),
('temperature_out_celsius', round(measured['temperature_out_celsius'][i], p)),
('humidity_in_relative_percent', round(measured['rh_in_percentage'][i], p)),
('humidity_in_absolute_g_m3', round(measured['rh_in_absolute_g_m3'][i], p)),
('humidity_in_specific_g_kg', round(measured['rh_in_specific_g_kg'][i], p)),
('humidity_out_relative_percent', round(measured['rh_out_percentage'][i], p)),
('humidity_out_absolute_g_m3', round(measured['rh_out_absolute_g_m3'][i], p)),
('humidity_out_specific_g_kg', round(measured['rh_out_specific_g_kg'][i], p)),
('pressure_in_hpa', round(measured['pressure_in_hpa'][i], p)),
('temperature_celsius_difference', round(temperature_diff, p)),
('humidity_relative_percent_difference', round(humidity_rh_diff, p)),
('humidity_absolute_g_m3_difference', round(humidity_abs_diff, p)),
('humidity_specific_g_kg_difference', round(humidity_spec_diff, p)),
])
def __read(self):
event_types = {}
with open(self.__filename, mode='r') as csv_file:
csv_reader = csv.DictReader(csv_file, delimiter=',')
for row in csv_reader:
record = {
'datetime': int(DateTimeUtil.local_time_str_to_utc(row['datetime']).timestamp()),
'readable': row['datetime'],
'event': row['event'],
'prediction': row['prediction(event)'],
'valid': row['valid']
}
if record['prediction'] == '':
record['valid'] = 'no'
if row['valid'] == 'no':
self.count -= 1
if record['prediction'] == '' and record['event'] == 'nothing':
continue
self.__data.append(record)
event_types[row['event']] = None
self.count += len(self.__data)
if len(event_types) == 2:
if 'open' in event_types and 'nothing' in event_types:
self.__event_type = 'open'
elif 'close' in event_types and 'nothing' in event_types:
self.__event_type = 'close'
else:
raise ValueError('%s must contains only 2 types of event column')
elif len(event_types) == 1 and 'nothing' in event_types:
self.__event_type = 'open'
else:
raise ValueError('%s must contains only 2 types of event column')
def db_to_simple_graph(event, column, color, label, number_output_records):
x = []
y = []
length = len(event['measured'][column])
step = 1
if number_output_records is not None:
step = length // number_output_records
# ak je step nula, znamena to, ze nie je dostatok udajov, vykreslime
# vsetky dostupne data so step jedna
if step == 0:
step = 1
if step > 1:
step += 1
start = event['e_start']['timestamp'] + event['start_shift']
for i in range(0, length):
value = event['measured'][column][i]
if i % step != 0:
continue
timestamp = start + i
x.append(DateTimeUtil.utc_timestamp_to_str(timestamp, '%H:%M:%S'))
if value is None:
y.append('Null')
else:
y.append(value)
return {
'timestamps': x,
'values': y,
'label_x': label,
'color': color,
'open_close': column == 'open_close',
}
def training_data_without_opposite(con, table_name, events, func,
row_selector, interval_selector):
attrs = []
for k in range(0, len(events)):
event = events[k]
start = event['e_start']['timestamp']
end = event['e_end']['timestamp']
try:
data1 = func(con, table_name, start, row_selector,
interval_selector, end)
time = DateTimeUtil.utc_timestamp_to_str(
start, '%Y/%m/%d %H:%M:%S')
data1.insert(0, ('datetime', time))
attrs.append(OrderedDict(data1))
except Exception as e:
# logging.error(str(e))
continue
return attrs
def prepare_derivation(con, events: list, intervals_before: list,
intervals_after: list, table_name: str,
precision: int, derivation_attr_name: str,
derivation_index: int):
for i in range(0, len(events)):
event = events[i]
no_event_shift = event['no_event_time_shift']
start = event['e_start']['timestamp']
open_value = event['measured'][derivation_attr_name][0]
no_event_open_value = event['no_event_values'][derivation_index]
if no_event_open_value is None:
t = DateTimeUtil.utc_timestamp_to_str(start + no_event_shift,
'%Y-%m-%d %H:%M:%S')
logging.warning('no_event value is None: %s' % t)
# derivacia pred otvorenim okna
# generovanie derivacii medzi hodnou otvorenia okna a hodnotou niekde
# v minulosti, ktora je posunuta o zadany interval dozadu
for interval in intervals_before:
value_time = start - interval
value = Storage.one_row(con, table_name, derivation_attr_name,
value_time)
derivation = None
if value is not None and value[0] is not None:
derivation = round(
(open_value - float(value[0])) / interval, precision)
event['derivation']['before'].append(derivation)
# derivacia po otvoreni okna
# generovanie derviacii medzi hodnotou otvorenia okna a hodnotou niekde,
# v buducnosti, ktora je posunuta o zadany interval dopredu
for interval in intervals_after:
value_time = start + interval
value = Storage.one_row(con, table_name, derivation_attr_name,
value_time)
derivation = None
if value is not None and value[0] is not None:
derivation = round(
(float(value[0]) - open_value) / interval, precision)
event['derivation']['after'].append(derivation)
# derivacia pred no_event
# generovanie derivacii medzi hodnou otvorenia okna a hodnotou niekde
# v minulostia, ktora je posunuta o zadany interval dozadu
# tento cas je posunuty este aj o posun danej udalosti
for interval in intervals_before:
value_time = start + no_event_shift - interval
value = Storage.one_row(con, table_name, derivation_attr_name,
value_time)
derivation = None
if value is not None and value[
0] is not None and no_event_open_value is not None:
derivation = round(
(float(no_event_open_value) - float(value[0])) /
interval, precision)
else:
event['valid_event'] = False
event['derivation']['no_event_before'].append(derivation)
# derivacia pred po no_event
# generovanie derivacii medzi hodnou otvorenia okna a hodnotou niekde
# v minulostia, ktora je posunuta o zadany interval dozadu
# tento cas je posunuty este aj o posun danej udalosti
for interval in intervals_after:
value_time = start + no_event_shift + interval
value = Storage.one_row(con, table_name, derivation_attr_name,
value_time)
derivation = None
if value is not None and value[
0] is not None and no_event_open_value is not None:
derivation = round(
(float(value[0]) - float(no_event_open_value)) /
interval, precision)
else:
event['valid_event'] = False
event['derivation']['no_event_after'].append(derivation)
event['derivation']['intervals_before'] = intervals_before
event['derivation']['intervals_after'] = intervals_after
event['derivation']['intervals_no_event_before'] = intervals_before
event['derivation']['intervals_no_event_after'] = intervals_after
return events
def generate_file(con, start_shift, end_shift, output_file):
logging.info('start: ' + output_file)
graphs = Graph("./../../src/graph")
# download data
storage = Storage('examples/events_klarka_shower.json', 0, 'measured_klarka_shower')
d = storage.load_data(con, start_shift, end_shift, 'temperature_in_celsius')
logging.info('downloaded events: %d' % len(d))
# apply filters to events
filtered = FilterUtil.only_valid_events(d)
# for travis
if ConnectionUtil.is_testable_system():
filtered = filtered[:ConnectionUtil.MAX_TESTABLE_EVENTS]
logging.info('events after applying the filter: %d' % len(filtered))
fill_start_end(filtered)
logging.info('start generating graphs')
gr = []
for event in filtered:
t = DateTimeUtil.utc_timestamp_to_str(event['e_start']['timestamp'], '%d.%m. %H:%M:%S')
t += ' - '
t += DateTimeUtil.utc_timestamp_to_str(event['e_end']['timestamp'], '%H:%M:%S')
g = {
'title': t,
'group': 'one',
'graphs': [
Graph.db_to_simple_graph(event, 'temperature_in_celsius', 'blue',
'Temperature', 75),
Graph.db_to_simple_graph(event, 'open_close', 'orange', 'Open', 75),
]
}
gr.append(g)
g = {
'title': t,
'group': 'two',
'graphs': [
Graph.db_to_simple_graph(event, 'rh_in_percentage', 'red',
'Relative humidity [%]', 75),
Graph.db_to_simple_graph(event, 'open_close', 'orange', 'Open', 75),
]
}
gr.append(g)
g = {
'title': t,
'group': 'tree',
'graphs': [
Graph.db_to_simple_graph(event, 'rh_in_absolute_g_m3', 'green',
'Absolute humidity [g/m3]', 75),
Graph.db_to_simple_graph(event, 'open_close', 'orange', 'Open', 75),
]
}
gr.append(g)
g = {
'title': t,
'group': 'four',
'graphs': [
Graph.db_to_simple_graph(event, 'rh_in_specific_g_kg', 'purple',
'Specific humidity [g/kg]', 75),
Graph.db_to_simple_graph(event, 'open_close', 'orange', 'Open', 75),
]
}
gr.append(g)
graphs.gen(gr, output_file + '.html', 0, 0, global_range=True)
logging.info('end generating graphs')
last_time_notification = None
try:
while not loop_end:
time.sleep(SLEEP_TIME)
sensor_info = cl.sensors_info(GATEWAY_ID, DEVICE_ID)
value = extract_value(sensor_info, SENSOR_ID)
measured_time = value['at']
measured_value = value['value']
if measured_value is None:
continue
time_out_str = DateTimeUtil.utc_timestamp_to_str(measured_time, '%Y/%m/%d %H:%M:%S')
logging.debug(time_out_str + ': ' + measured_value)
text = 'Aktualna koncentracia {0} ppm'.format(int(float(measured_value)))
if last_time_notification is None or last_time_notification + DELAY_BETWEEN_SEND < time.time():
if float(measured_value) > CO2_LIMIT:
text += ", (limit " + str(CO2_LIMIT) + " ppm), prosim vyvetrajte."
last_time_notification = time.time()
send_notification(text)
logging.debug('send notification')
os.remove(CONFIG_FILE)
except KeyboardInterrupt:
def training_data(con,
table_name,
events,
func,
row_selector,
interval_selector,
event_type,
print_each=10):
"""Generation of training data.
:param con:
:param table_name: table name
:param events: event list
:param func:
:param row_selector:
:param interval_selector:
:return:
"""
training_events = []
attrs = []
for k in range(0, len(events)):
event = events[k]
start = event['e_start']['timestamp']
end = event['e_end']['timestamp']
no_event_start = start + event['no_event_time_shift']
no_event_end = end - event['no_event_time_shift']
if k % print_each == 0:
logging.debug('{0}/{1} events'.format(k, len(events)))
if event_type == 'open':
event_time = start
no_event_time = no_event_start
elif event_type == 'close':
event_time = end
no_event_time = no_event_end
else:
raise ValueError('event type must be: open or close')
try:
data1 = func(con, table_name, event_time, row_selector,
interval_selector)
data2 = func(con, table_name, no_event_time, row_selector,
interval_selector)
time = DateTimeUtil.utc_timestamp_to_str(
event_time, '%Y/%m/%d %H:%M:%S')
data1.insert(0, ('datetime', time))
data1.insert(1, ('event', event_type))
attrs.append(OrderedDict(data1))
no_time = DateTimeUtil.utc_timestamp_to_str(
no_event_time, '%Y/%m/%d %H:%M:%S')
data2.insert(0, ('datetime', no_time))
data2.insert(1, ('event', 'nothing'))
attrs.append(OrderedDict(data2))
training_events.append(event)
except Exception as e:
# logging.error(str(e))
continue
return attrs, training_events
def testing_data(con,
table_name,
start,
end,
write_each,
func,
row_selector,
interval_selector,
event_type,
log_every_hour=3):
"""Generation of testing data.
:param con:
:param table_name: table name
:param start: interval from which testing data is generated
:param end: interval to which testing data is generated
:param write_each:
:param func:
:param row_selector:
:param interval_selector:
:return:
"""
attrs = []
bad_open_type_events = []
global DATA_CACHE
for t in range(start, end):
if t % (log_every_hour * 3600) == 0:
logging.debug(DateTimeUtil.utc_timestamp_to_str(t))
act_row = None
if act_row is None:
previous_row = Storage.one_row(con, table_name, 'open_close',
t - 1)
else:
previous_row = act_row
act_row = Storage.one_row(con, table_name, 'open_close', t)
if event_type not in ['open', 'close']:
raise ValueError('event type must be: open or close')
if previous_row is None or act_row is None:
continue
open_state = 'nothing'
if event_type == 'open' and previous_row[0] == 0 and act_row[
0] == 1:
open_state = event_type
elif event_type == 'close' and previous_row[0] == 1 and act_row[
0] == 0:
open_state = event_type
if open_state == 'nothing':
if t % write_each != 0:
continue
try:
DATA_CACHE = func(con, table_name, t, row_selector,
interval_selector)
except Exception as e:
# logging.error(str(e))
if open_state in ['open', 'close']:
bad_open_type_events.append(t)
continue
time = DateTimeUtil.utc_timestamp_to_str(t, '%Y/%m/%d %H:%M:%S')
DATA_CACHE.insert(0, ('datetime', time))
DATA_CACHE.insert(1, ('event', open_state))
DATA_CACHE.append(('valid', 'yes'))
attrs.append(OrderedDict(DATA_CACHE))
if DATA_CACHE is None:
logging.warning('any {0} events can be skipped'.format(event_type))
else:
tmp = {}
for item in DATA_CACHE:
key = item[0]
tmp[key] = None
tmp['event'] = event_type
tmp['valid'] = 'no'
for timestamp in bad_open_type_events:
tmp['datetime'] = DateTimeUtil.utc_timestamp_to_str(
timestamp, '%Y/%m/%d %H:%M:%S')
attrs.append(OrderedDict(tmp))
return attrs
def with_delay(self, before, after):
nothing_as_true_nothing = 0
open_as_true_nothing = 0
open_as_true_open = 0
nothing_as_true_open = 0
extended = {}
invalid = {}
wrong_prediction = []
if self.__data == []:
self.__read()
intervals = []
for row in self.__data:
if row['event'] == self.__event_type:
t = row['datetime']
intervals.append((t - before, t, t + after))
intervals.sort()
for i in range(1, len(intervals)):
if intervals[i - 1][2] > intervals[i][0]:
t1 = DateTimeUtil.utc_timestamp_to_str(intervals[i - 1][1], '%d.%m. %H:%M:%S')
t2 = DateTimeUtil.utc_timestamp_to_str(intervals[i][1], '%d.%m. %H:%M:%S')
print('prekryvajuce sa intervaly {0} a {1}'.format(t1, t2))
for row in intervals:
extended[row[1]] = []
invalid[row[1]] = []
for row in self.__data:
found = False
for interval in intervals:
if interval[0] < row['datetime'] < interval[2]:
extended[interval[1]].append(row['prediction'])
invalid[interval[1]].append(row['valid'])
found = True
if found or row['valid'] == 'no':
continue
if row['event'] == row['prediction']:
if row['event'] == self.__event_type:
open_as_true_open += 1
elif row['event'] == 'nothing':
nothing_as_true_nothing += 1
else:
raise ValueError('error')
else:
if row['event'] == 'nothing' and row['prediction'] == self.__event_type:
open_as_true_nothing += 1
if row['event'] != self.__event_type:
wrong_prediction.append(row['readable'])
elif row['prediction'] != '':
nothing_as_true_open += 1
for key, interval in extended.items():
if len(interval) == 1 and 'no' in invalid[key]:
continue
found = False
for k in range(0, len(interval)):
row = interval[k]
if row == self.__event_type:
found = True
break
if found:
if 'no' not in invalid[key]:
open_as_true_open += 1
else:
if 'no' not in invalid[key]:
nothing_as_true_open += 1
nothing_as_true_nothing += len(interval) - 1
res = {
'records': self.count,
'accuracy': round(((nothing_as_true_nothing + open_as_true_open) / self.count) * 100, 2),
'nothing_as_true_nothing': nothing_as_true_nothing,
'open_as_true_nothing': open_as_true_nothing,
'open_as_true_open': open_as_true_open,
'nothing_as_true_open': nothing_as_true_open,
}
res['sum'] = res['nothing_as_true_nothing'] + res['open_as_true_nothing'] + res['open_as_true_open'] \
+ res['nothing_as_true_open']
return self.__simple_table(res), wrong_prediction, res
testing_set('measured_filtered_peto', start, end,
'{0}/gt_peto.csv'.format(directory))
# Klarka
start = int(
DateTimeUtil.local_time_str_to_utc('2018/12/18 18:00:00').timestamp())
testing_set('measured_klarka', start, end,
'{0}/gt_klarka.csv'.format(directory))
if __name__ == '__main__':
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(levelname)s %(message)s')
# tabulka s CO2, ktora neprekroci hranicu 2000ppm
table_name = 'measured_filtered_peto'
directory = 'co2_t_h'
if not os.path.isdir(directory):
os.mkdir(directory)
training_set('examples/events_peto.json', -500, table_name, directory)
start = int(
DateTimeUtil.local_time_str_to_utc('2018/10/07 06:00:00').timestamp())
testing_set(table_name, start, start + 100,
'{0}/testing.csv'.format(directory))
# testing_month(table_name, start)
# generic_testing(directory)
def generate_graphs_sensor_1(event: dict, owner: str,
number_output_records: int):
"""Generates graph based on data measured using sensor 1.
:param event: basic information about event
:param owner: sensor owner(klarka|peto), name must be the same as in database
:param number_output_records: number of points that are required in graph
:return: graph that can contain several graphs
"""
n = number_output_records
graphs = []
t = DateTimeUtil.utc_timestamp_to_str(event['e_start']['timestamp'],
'%d.%m. %H:%M:%S')
t += ' - '
t += DateTimeUtil.utc_timestamp_to_str(event['e_end']['timestamp'],
'%H:%M:%S')
g = {
'title':
'Temp in and out ' + t,
'stat':
generate_info(event, owner),
'graphs': [
Graph.db_to_simple_graph(event, 'temperature_in_celsius',
'DarkRed', 'temp in', n),
Graph.db_to_simple_graph(event, 'temperature_out_celsius',
'LightCoral', 'temp out', n)
]
}
graphs.append(g)
g = {
'title':
'Relative hum in and out ' + t,
'graphs': [
Graph.db_to_simple_graph(event, 'rh_in_percentage', 'blue',
'hum in', n),
Graph.db_to_simple_graph(event, 'rh_out_percentage', 'red',
'hum out', n),
]
}
graphs.append(g)
g = {
'title':
'Specific hum in and out ' + t,
'graphs': [
Graph.db_to_simple_graph(event, 'rh_in_specific_g_kg', 'blue',
'hum in', n),
Graph.db_to_simple_graph(event, 'rh_out_specific_g_kg', 'red',
'hum out', n),
]
}
graphs.append(g)
g = {
'title':
'Absolute hum in and out ' + t,
'graphs': [
Graph.db_to_simple_graph(event, 'rh_in_absolute_g_m3', 'blue',
'hum in', n),
Graph.db_to_simple_graph(event, 'rh_out_absolute_g_m3', 'red',
'hum out', n),
]
}
graphs.append(g)
return graphs
# minimum
ax.set_ylim(min(y1) - 50, max(y1)+50)
ax.set_xlim(dt.datetime.fromtimestamp(raw_t[0]-5), dt.datetime.fromtimestamp(raw_t[-1]+5))
filename = simple_graph.__name__ + '.eps'
fig.canvas.set_window_title(filename)
# nastavenie, aby sa aj pri malej figsize zobrazoval nazov X osy
plt.tight_layout()
fig.savefig(filename, bbox_inches='tight', pad_inches=0)
if __name__ == '__main__':
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(levelname)s %(message)s')
con = ConnectionUtil.create_con()
start = int(DateTimeUtil.local_time_str_to_utc('2018/10/07 06:00:00').timestamp())
end = int(DateTimeUtil.local_time_str_to_utc('2018/10/07 09:00:00').timestamp())
table_name = 'measured_filtered_peto'
all = Storage.dw_columns_ordered(con, start, end, 'measured_time,co2_in_ppm', table_name)
CSVUtil.create_csv_file(all, 'test.csv')
simple_graph('test.csv')
# plt.show()
def read_meta(self):
with open(self.__filename) as f:
events = json.load(f)
out = []
for event in events['events']:
# atributy, ktore su spolocne pre vsetky udalosti
attributes = {
'e_start': {
'readable':
event['times']['event_start'],
'timestamp':
int(
DateTimeUtil.local_time_str_to_utc(
event['times']['event_start']).timestamp())
},
'e_end': {
'readable':
event['times']['event_end'],
'timestamp':
int(
DateTimeUtil.local_time_str_to_utc(
event['times']['event_end']).timestamp())
},
'measured': {
'pressure_in_hpa': [],
'temperature_in_celsius': [],
'temperature_in2_celsius': [],
'temperature_out_celsius': [],
'rh_in_percentage': [],
'rh_in_absolute_g_m3': [],
'rh_in_specific_g_kg': [],
'rh_in2_percentage': [],
'rh_in2_absolute_g_m3': [],
'rh_in2_specific_g_kg': [],
'rh_out_percentage': [],
'rh_out_absolute_g_m3': [],
'rh_out_specific_g_kg': [],
'co2_in_ppm': []
},
'derivation': {
'after': [],
'before': [],
'no_event_after': [],
'no_event_before': []
},
'no_event_time_shift': self.__no_event_time_shift,
'no_event_values': None,
'valid_event': True
}
start = attributes['e_start']['timestamp']
end = attributes['e_end']['timestamp']
attributes['event_duration'] = end - start
# doplnenie atributov, ktore su specificke pre dany json
# len sa nakopuruju jednotlive polozky json struktury
for key, value in event.items():
if key in ['times', 'devices']:
continue
attributes[key] = value
out.append(attributes)
return out
logging.info('end')
def testing_month(table_name, start):
mesiac = 30 * 24 * 3600
file_names = [
'1_listopad.csv',
'2_prosinec.csv',
'3_leden.csv',
'4_unor.csv',
]
for file_name in file_names:
testing_set(table_name, start, start + mesiac, file_name)
start += mesiac
if __name__ == '__main__':
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(levelname)s %(message)s')
table_name = 'measured_klarka_shower'
training_set('examples/events_klarka_shower.json', -500, table_name)
start = int(DateTimeUtil.local_time_str_to_utc('2018/11/01 05:30:00').timestamp())
testing_set(table_name, start, start + 100, 'testing.csv')
# testing_month(table_name, start)
def main(events_file: str, start_shift: int, end_shift: int,
output_filename: str, output_records: int):
logging.info('start')
graphs = Graph("./../../src/graph")
# download data
con = ConnectionUtil.create_con()
storage = Storage(events_file, 0, 'measured_klarka')
d = storage.load_data(con, start_shift, end_shift,
'temperature_in_celsius')
logging.info('downloaded events: %d' % len(d))
# apply filters to events
filtered = FilterUtil.only_valid_events(d)
filtered = FilterUtil.temperature_diff(filtered, 5, 100)
filtered = FilterUtil.temperature_out_max(filtered, 15)
filtered = FilterUtil.humidity(filtered, 6, 1.6, 100)
min_timestamp = int(
DateTimeUtil.local_time_str_to_utc('2018/11/01 00:01:00').timestamp())
filtered = FilterUtil.min_timestamp(filtered, min_timestamp)
filtered = FilterUtil.min_max_time_interval(filtered, 1440, 1620)
# for travis
if ConnectionUtil.is_testable_system():
filtered = filtered[:ConnectionUtil.MAX_TESTABLE_EVENTS]
logging.info('events after applying the filter: %d' % len(filtered))
# data for graph generation measured using sensor 1
sensor1_events = filtered
logging.info('event count: %d for senzor 1' % len(sensor1_events))
linear_reg(sensor1_events, 'rh_in_specific_g_kg', 'linear1_sh')
linear_reg(sensor1_events, 'rh_in_absolute_g_m3', 'linear1_ah')
linear_reg(sensor1_events, 'temperature_in_celsius', 'linear1_temp')
# graph generation - sensor 1
logging.info('start generating graphs of events from sensor 1')
graphs_sensor_1 = []
for event in sensor1_events:
graphs_sensor_1 += gen_graphs(event, output_records, [
'rh_in_specific_g_kg', 'rh_in_absolute_g_m3',
'temperature_in_celsius'
], ['linear1_sh', 'linear1_ah', 'linear1_temp'])
graphs.gen(graphs_sensor_1,
'sensor1_' + output_filename,
0,
0,
global_range=True)
logging.info('end generating graphs of events from sensor 1')
# data for graph generation measured using sensor 2
sensor2_events = filtered
logging.info('event count: %d for sensor 2' % len(sensor2_events))
sensor2_events = FilterUtil.measured_values_not_empty(
sensor2_events, 'rh_in2_specific_g_kg')
sensor2_events = FilterUtil.measured_values_not_empty(
sensor2_events, 'rh_in2_absolute_g_m3')
sensor2_events = FilterUtil.measured_values_not_empty(
sensor2_events, 'temperature_in2_celsius')
logging.info('events after applying the filter: %d' % len(sensor2_events))
linear_reg(sensor2_events, 'rh_in2_specific_g_kg', 'linear2_sh')
linear_reg(sensor2_events, 'rh_in2_absolute_g_m3', 'linear2_ah')
linear_reg(sensor2_events, 'temperature_in2_celsius', 'linear2_temp')
humidity_info_csv(sensor2_events, start_shift, end_shift)
# graph generation - sensor 2
logging.info('start generating graphs of events from sensor 2')
graphs_sensor_2 = []
for event in sensor2_events:
graphs_sensor_2 += gen_graphs(event, output_records, [
'rh_in2_specific_g_kg', 'rh_in2_absolute_g_m3',
'temperature_in2_celsius'
], ['linear2_sh', 'linear2_ah', 'linear2_temp'])
graphs.gen(graphs_sensor_2,
'sensor2_' + output_filename,
0,
0,
global_range=True)
logging.info('end generating graphs of events from sensor 2')
logging.info('end')
def gen_graphs(event: dict, number_output_records: int, attr_name: list,
lin_reg_attr_name: list):
"""Generates graph based on data measured using sensor 1.
:param event: basic information about event
:param number_output_records: number of points that are required in graph
:param attr_name:
:param lin_reg_attr_name:
:return: graph that can contain several graphs
"""
n = number_output_records
graphs = []
t = DateTimeUtil.utc_timestamp_to_str(event['e_start']['timestamp'],
'%d.%m. %H:%M:%S')
t += ' - '
t += DateTimeUtil.utc_timestamp_to_str(event['e_end']['timestamp'],
'%H:%M:%S')
g = {
'title':
'Specific hum in and out ' + t,
'group':
'one',
'graphs': [
Graph.db_to_simple_graph(event, attr_name[0], 'blue', 'sh in', n),
Graph.db_to_simple_graph(event, 'rh_out_specific_g_kg', 'red',
'sh out', n),
Graph.db_to_simple_graph(event, lin_reg_attr_name[0], 'orange',
'sh lin reg', n),
]
}
graphs.append(g)
g = {
'title':
'Absolute hum in and out ' + t,
'group':
'one',
'graphs': [
Graph.db_to_simple_graph(event, attr_name[1], 'blue', 'ah in', n),
Graph.db_to_simple_graph(event, 'rh_out_absolute_g_m3', 'red',
'ah out', n),
Graph.db_to_simple_graph(event, lin_reg_attr_name[1], 'orange',
'ah lin reg', n),
]
}
graphs.append(g)
g = {
'title':
'Temperature in and out ' + t,
'group':
'one',
'graphs': [
Graph.db_to_simple_graph(event, attr_name[2], 'blue', 'temp in',
n),
Graph.db_to_simple_graph(event, 'temperature_out_celsius', 'red',
'temp out', n),
Graph.db_to_simple_graph(event, lin_reg_attr_name[2], 'orange',
'temp lin reg', n),
]
}
graphs.append(g)
return graphs
interval = intervals[k]
interval['avg'] = round(
(interval['to'] - interval['from']) / 2 + interval['from'], 3)
out = []
with open(output_example_set, mode='r') as csv_file:
csv_reader = csv.DictReader(csv_file, delimiter=',')
for row in csv_reader:
predicted_param = extract_interval(
row['prediction(Regression_co2_in_ppm_before_0)'], default_min,
default_max)
record = {
'datetime':
int(
DateTimeUtil.local_time_str_to_utc(
row['datetime']).timestamp()),
'readable':
row['datetime'],
'event':
extract_interval(row['Regression_co2_in_ppm_before_0'],
default_min, default_max),
'prediction':
predicted_param,
'measured':
float(row['actual_value']),
'co2_start':
float(row['co2_start']),
}
out.append(record)
def prepare_derivation_lin_reg(con, events: list, intervals_before: list,
intervals_after: list, table_name: str,
precision: int, derivation_attr_name: str,
lin_window_size: tuple):
for i in range(0, len(events)):
event = events[i]
no_event_shift = event['no_event_time_shift']
start = event['e_start']['timestamp']
values1 = Storage.select_interval(con, start - lin_window_size[0],
start + lin_window_size[1],
derivation_attr_name, table_name)
values2 = Storage.select_interval(
con, start + no_event_shift - lin_window_size[0],
start + no_event_shift + lin_window_size[1],
derivation_attr_name, table_name)
if not values2 or not values1:
event['valid_event'] = False
continue
open_value = lin_reg(values1, lin_window_size[0] - 1)
no_event_open_value = lin_reg(values2, lin_window_size[0] - 1)
if no_event_open_value is None:
t = DateTimeUtil.utc_timestamp_to_str(start + no_event_shift,
'%Y-%m-%d %H:%M:%S')
logging.warning('no_event value is None: %s' % t)
# derivacia pred otvorenim okna
# generovanie derivacii medzi hodnou otvorenia okna a hodnotou niekde
# v minulosti, ktora je posunuta o zadany interval dozadu
for interval in intervals_before:
value_time = start - interval
v = Storage.select_interval(con,
value_time - lin_window_size[0],
value_time + lin_window_size[1],
derivation_attr_name, table_name)
if not v:
event['valid_event'] = False
break
value = lin_reg(v, lin_window_size[0] - 1)
derivation = None
if value is not None and value is not None:
derivation = round((open_value - value) / interval,
precision)
event['derivation']['before'].append(derivation)
# derivacia po otvoreni okna
# generovanie derviacii medzi hodnotou otvorenia okna a hodnotou niekde,
# v buducnosti, ktora je posunuta o zadany interval dopredu
for interval in intervals_after:
value_time = start + interval
v = Storage.select_interval(con,
value_time - lin_window_size[0],
value_time + lin_window_size[1],
derivation_attr_name, table_name)
if not v:
event['valid_event'] = False
break
value = lin_reg(v, lin_window_size[0] - 1)
derivation = None
if value is not None and value is not None:
derivation = round((value - open_value) / interval,
precision)
event['derivation']['after'].append(derivation)
# derivacia pred no_event
# generovanie derivacii medzi hodnou otvorenia okna a hodnotou niekde
# v minulostia, ktora je posunuta o zadany interval dozadu
# tento cas je posunuty este aj o posun danej udalosti
for interval in intervals_before:
value_time = start + no_event_shift - interval
v = Storage.select_interval(con,
value_time - lin_window_size[0],
value_time + lin_window_size[1],
derivation_attr_name, table_name)
if not v:
event['valid_event'] = False
break
value = lin_reg(v, lin_window_size[0] - 1)
derivation = None
if value is not None and value is not None and no_event_open_value is not None:
derivation = round(
(no_event_open_value - value) / interval, precision)
else:
event['valid_event'] = False
event['derivation']['no_event_before'].append(derivation)
# derivacia pred po no_event
# generovanie derivacii medzi hodnou otvorenia okna a hodnotou niekde
# v minulostia, ktora je posunuta o zadany interval dozadu
# tento cas je posunuty este aj o posun danej udalosti
for interval in intervals_after:
value_time = start + no_event_shift + interval
v = Storage.select_interval(con,
value_time - lin_window_size[0],
value_time + lin_window_size[1],
derivation_attr_name, table_name)
if not v:
event['valid_event'] = False
break
value = lin_reg(v, lin_window_size[0] - 1)
derivation = None
if value is not None and value is not None and no_event_open_value is not None:
derivation = round(
(value - no_event_open_value) / interval, precision)
else:
event['valid_event'] = False
event['derivation']['no_event_after'].append(derivation)
event['derivation']['intervals_before'] = intervals_before
event['derivation']['intervals_after'] = intervals_after
event['derivation']['intervals_no_event_before'] = intervals_before
event['derivation']['intervals_no_event_after'] = intervals_after