def generic_testing(directory):
end = int(
DateTimeUtil.local_time_str_to_utc('2019/04/29 18:00:00').timestamp())
# David
start = int(
DateTimeUtil.local_time_str_to_utc('2019/04/03 18:00:00').timestamp())
testing_set('measured_david', start, end,
'{0}/gt_david.csv'.format(directory))
# Martin
start = int(
DateTimeUtil.local_time_str_to_utc('2019/04/01 18:00:00').timestamp())
testing_set('measured_martin', start, end,
'{0}/gt_martin.csv'.format(directory))
# Peto , februar, marec, april
start = int(
DateTimeUtil.local_time_str_to_utc('2019/02/04 18:00:00').timestamp())
testing_set('measured_filtered_peto', start, end,
'{0}/gt_peto.csv'.format(directory))
# Klarka
start = int(
DateTimeUtil.local_time_str_to_utc('2018/12/18 18:00:00').timestamp())
testing_set('measured_klarka', start, end,
'{0}/gt_klarka.csv'.format(directory))
def additional_training_set(con,
table_name,
no_event_records,
func,
row_selector,
interval_selector,
print_each=10):
"""Supplementary generation of training data based on given time points.
:param con:
:param table_name: table name
:param no_event_records: list of pairs for attribute generation
:param func:
:param row_selector:
:param interval_selector:
:return:
"""
attrs = []
for k in range(0, len(no_event_records)):
row = no_event_records[k]
if k % print_each == 0:
logging.debug('{0}/{1} events'.format(k,
len(no_event_records)))
if row[0] == '':
logging.warning('empty row in additional sets')
continue
start = int(DateTimeUtil.local_time_str_to_utc(row[0]).timestamp())
try:
data1 = func(con, table_name, start, row_selector,
interval_selector)
time = DateTimeUtil.utc_timestamp_to_str(
start, '%Y/%m/%d %H:%M:%S')
data1.insert(0, ('datetime', time))
data1.insert(1, ('event', row[1]))
attrs.append(OrderedDict(data1))
except Exception as e:
logging.error(str(e))
continue
return attrs
def __read(self):
event_types = {}
with open(self.__filename, mode='r') as csv_file:
csv_reader = csv.DictReader(csv_file, delimiter=',')
for row in csv_reader:
record = {
'datetime': int(DateTimeUtil.local_time_str_to_utc(row['datetime']).timestamp()),
'readable': row['datetime'],
'event': row['event'],
'prediction': row['prediction(event)'],
'valid': row['valid']
}
if record['prediction'] == '':
record['valid'] = 'no'
if row['valid'] == 'no':
self.count -= 1
if record['prediction'] == '' and record['event'] == 'nothing':
continue
self.__data.append(record)
event_types[row['event']] = None
self.count += len(self.__data)
if len(event_types) == 2:
if 'open' in event_types and 'nothing' in event_types:
self.__event_type = 'open'
elif 'close' in event_types and 'nothing' in event_types:
self.__event_type = 'close'
else:
raise ValueError('%s must contains only 2 types of event column')
elif len(event_types) == 1 and 'nothing' in event_types:
self.__event_type = 'open'
else:
raise ValueError('%s must contains only 2 types of event column')
interval = intervals[k]
interval['avg'] = round(
(interval['to'] - interval['from']) / 2 + interval['from'], 3)
out = []
with open(output_example_set, mode='r') as csv_file:
csv_reader = csv.DictReader(csv_file, delimiter=',')
for row in csv_reader:
predicted_param = extract_interval(
row['prediction(Regression_co2_in_ppm_before_0)'], default_min,
default_max)
record = {
'datetime':
int(
DateTimeUtil.local_time_str_to_utc(
row['datetime']).timestamp()),
'readable':
row['datetime'],
'event':
extract_interval(row['Regression_co2_in_ppm_before_0'],
default_min, default_max),
'prediction':
predicted_param,
'measured':
float(row['actual_value']),
'co2_start':
float(row['co2_start']),
}
out.append(record)
testing_set('measured_filtered_peto', start, end,
'{0}/gt_peto.csv'.format(directory))
# Klarka
start = int(
DateTimeUtil.local_time_str_to_utc('2018/12/18 18:00:00').timestamp())
testing_set('measured_klarka', start, end,
'{0}/gt_klarka.csv'.format(directory))
if __name__ == '__main__':
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(levelname)s %(message)s')
# tabulka s CO2, ktora neprekroci hranicu 2000ppm
table_name = 'measured_filtered_peto'
directory = 'co2_t_h'
if not os.path.isdir(directory):
os.mkdir(directory)
training_set('examples/events_peto.json', -500, table_name, directory)
start = int(
DateTimeUtil.local_time_str_to_utc('2018/10/07 06:00:00').timestamp())
testing_set(table_name, start, start + 100,
'{0}/testing.csv'.format(directory))
# testing_month(table_name, start)
# generic_testing(directory)
def main(events_file: str, start_shift: int, end_shift: int,
output_filename: str, output_records: int):
logging.info('start')
graphs = Graph("./../../src/graph")
# download data
con = ConnectionUtil.create_con()
storage = Storage(events_file, 0, 'measured_klarka')
d = storage.load_data(con, start_shift, end_shift,
'temperature_in_celsius')
logging.info('downloaded events: %d' % len(d))
# apply filters to events
filtered = FilterUtil.only_valid_events(d)
filtered = FilterUtil.temperature_diff(filtered, 5, 100)
filtered = FilterUtil.temperature_out_max(filtered, 15)
filtered = FilterUtil.humidity(filtered, 6, 1.6, 100)
min_timestamp = int(
DateTimeUtil.local_time_str_to_utc('2018/11/01 00:01:00').timestamp())
filtered = FilterUtil.min_timestamp(filtered, min_timestamp)
filtered = FilterUtil.min_max_time_interval(filtered, 1440, 1620)
# for travis
if ConnectionUtil.is_testable_system():
filtered = filtered[:ConnectionUtil.MAX_TESTABLE_EVENTS]
logging.info('events after applying the filter: %d' % len(filtered))
# data for graph generation measured using sensor 1
sensor1_events = filtered
logging.info('event count: %d for senzor 1' % len(sensor1_events))
linear_reg(sensor1_events, 'rh_in_specific_g_kg', 'linear1_sh')
linear_reg(sensor1_events, 'rh_in_absolute_g_m3', 'linear1_ah')
linear_reg(sensor1_events, 'temperature_in_celsius', 'linear1_temp')
# graph generation - sensor 1
logging.info('start generating graphs of events from sensor 1')
graphs_sensor_1 = []
for event in sensor1_events:
graphs_sensor_1 += gen_graphs(event, output_records, [
'rh_in_specific_g_kg', 'rh_in_absolute_g_m3',
'temperature_in_celsius'
], ['linear1_sh', 'linear1_ah', 'linear1_temp'])
graphs.gen(graphs_sensor_1,
'sensor1_' + output_filename,
0,
0,
global_range=True)
logging.info('end generating graphs of events from sensor 1')
# data for graph generation measured using sensor 2
sensor2_events = filtered
logging.info('event count: %d for sensor 2' % len(sensor2_events))
sensor2_events = FilterUtil.measured_values_not_empty(
sensor2_events, 'rh_in2_specific_g_kg')
sensor2_events = FilterUtil.measured_values_not_empty(
sensor2_events, 'rh_in2_absolute_g_m3')
sensor2_events = FilterUtil.measured_values_not_empty(
sensor2_events, 'temperature_in2_celsius')
logging.info('events after applying the filter: %d' % len(sensor2_events))
linear_reg(sensor2_events, 'rh_in2_specific_g_kg', 'linear2_sh')
linear_reg(sensor2_events, 'rh_in2_absolute_g_m3', 'linear2_ah')
linear_reg(sensor2_events, 'temperature_in2_celsius', 'linear2_temp')
humidity_info_csv(sensor2_events, start_shift, end_shift)
# graph generation - sensor 2
logging.info('start generating graphs of events from sensor 2')
graphs_sensor_2 = []
for event in sensor2_events:
graphs_sensor_2 += gen_graphs(event, output_records, [
'rh_in2_specific_g_kg', 'rh_in2_absolute_g_m3',
'temperature_in2_celsius'
], ['linear2_sh', 'linear2_ah', 'linear2_temp'])
graphs.gen(graphs_sensor_2,
'sensor2_' + output_filename,
0,
0,
global_range=True)
logging.info('end generating graphs of events from sensor 2')
logging.info('end')
# minimum
ax.set_ylim(min(y1) - 50, max(y1)+50)
ax.set_xlim(dt.datetime.fromtimestamp(raw_t[0]-5), dt.datetime.fromtimestamp(raw_t[-1]+5))
filename = simple_graph.__name__ + '.eps'
fig.canvas.set_window_title(filename)
# nastavenie, aby sa aj pri malej figsize zobrazoval nazov X osy
plt.tight_layout()
fig.savefig(filename, bbox_inches='tight', pad_inches=0)
if __name__ == '__main__':
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(levelname)s %(message)s')
con = ConnectionUtil.create_con()
start = int(DateTimeUtil.local_time_str_to_utc('2018/10/07 06:00:00').timestamp())
end = int(DateTimeUtil.local_time_str_to_utc('2018/10/07 09:00:00').timestamp())
table_name = 'measured_filtered_peto'
all = Storage.dw_columns_ordered(con, start, end, 'measured_time,co2_in_ppm', table_name)
CSVUtil.create_csv_file(all, 'test.csv')
simple_graph('test.csv')
# plt.show()
logging.info('end')
def testing_month(table_name, start):
mesiac = 30 * 24 * 3600
file_names = [
'1_listopad.csv',
'2_prosinec.csv',
'3_leden.csv',
'4_unor.csv',
]
for file_name in file_names:
testing_set(table_name, start, start + mesiac, file_name)
start += mesiac
if __name__ == '__main__':
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(levelname)s %(message)s')
table_name = 'measured_klarka_shower'
training_set('examples/events_klarka_shower.json', -500, table_name)
start = int(DateTimeUtil.local_time_str_to_utc('2018/11/01 05:30:00').timestamp())
testing_set(table_name, start, start + 100, 'testing.csv')
# testing_month(table_name, start)
def read_meta(self):
with open(self.__filename) as f:
events = json.load(f)
out = []
for event in events['events']:
# atributy, ktore su spolocne pre vsetky udalosti
attributes = {
'e_start': {
'readable':
event['times']['event_start'],
'timestamp':
int(
DateTimeUtil.local_time_str_to_utc(
event['times']['event_start']).timestamp())
},
'e_end': {
'readable':
event['times']['event_end'],
'timestamp':
int(
DateTimeUtil.local_time_str_to_utc(
event['times']['event_end']).timestamp())
},
'measured': {
'pressure_in_hpa': [],
'temperature_in_celsius': [],
'temperature_in2_celsius': [],
'temperature_out_celsius': [],
'rh_in_percentage': [],
'rh_in_absolute_g_m3': [],
'rh_in_specific_g_kg': [],
'rh_in2_percentage': [],
'rh_in2_absolute_g_m3': [],
'rh_in2_specific_g_kg': [],
'rh_out_percentage': [],
'rh_out_absolute_g_m3': [],
'rh_out_specific_g_kg': [],
'co2_in_ppm': []
},
'derivation': {
'after': [],
'before': [],
'no_event_after': [],
'no_event_before': []
},
'no_event_time_shift': self.__no_event_time_shift,
'no_event_values': None,
'valid_event': True
}
start = attributes['e_start']['timestamp']
end = attributes['e_end']['timestamp']
attributes['event_duration'] = end - start
# doplnenie atributov, ktore su specificke pre dany json
# len sa nakopuruju jednotlive polozky json struktury
for key, value in event.items():
if key in ['times', 'devices']:
continue
attributes[key] = value
out.append(attributes)
return out