def row(self, column_name, time):
res = Storage.one_row(self.con, self.table_name, column_name, time)
if res is None or res[0] is None:
t = DateTimeUtil.utc_timestamp_to_str(time, '%Y/%m/%d %H:%M:%S')
raise ValueError('empty value at %s' % t)
return float(res[0])
def row(self, column_name, time):
if column_name not in self.cache:
self.cache[column_name] = {}
value = None
if time in self.cache[column_name]:
value = self.cache[column_name][time]
else:
res = Storage.one_row(self.con, self.table_name, column_name, time)
if res is not None and res[0] is not None:
self.cache[column_name][time] = float(res[0])
value = float(res[0])
if value is None:
t = DateTimeUtil.utc_timestamp_to_str(time, '%Y/%m/%d %H:%M:%S')
raise ValueError('empty value at %s' % t)
return value
def testing_data(con,
table_name,
start,
end,
write_each,
func,
row_selector,
interval_selector,
event_type,
log_every_hour=3):
"""Generation of testing data.
:param con:
:param table_name: table name
:param start: interval from which testing data is generated
:param end: interval to which testing data is generated
:param write_each:
:param func:
:param row_selector:
:param interval_selector:
:return:
"""
attrs = []
bad_open_type_events = []
global DATA_CACHE
for t in range(start, end):
if t % (log_every_hour * 3600) == 0:
logging.debug(DateTimeUtil.utc_timestamp_to_str(t))
act_row = None
if act_row is None:
previous_row = Storage.one_row(con, table_name, 'open_close',
t - 1)
else:
previous_row = act_row
act_row = Storage.one_row(con, table_name, 'open_close', t)
if event_type not in ['open', 'close']:
raise ValueError('event type must be: open or close')
if previous_row is None or act_row is None:
continue
open_state = 'nothing'
if event_type == 'open' and previous_row[0] == 0 and act_row[
0] == 1:
open_state = event_type
elif event_type == 'close' and previous_row[0] == 1 and act_row[
0] == 0:
open_state = event_type
if open_state == 'nothing':
if t % write_each != 0:
continue
try:
DATA_CACHE = func(con, table_name, t, row_selector,
interval_selector)
except Exception as e:
# logging.error(str(e))
if open_state in ['open', 'close']:
bad_open_type_events.append(t)
continue
time = DateTimeUtil.utc_timestamp_to_str(t, '%Y/%m/%d %H:%M:%S')
DATA_CACHE.insert(0, ('datetime', time))
DATA_CACHE.insert(1, ('event', open_state))
DATA_CACHE.append(('valid', 'yes'))
attrs.append(OrderedDict(DATA_CACHE))
if DATA_CACHE is None:
logging.warning('any {0} events can be skipped'.format(event_type))
else:
tmp = {}
for item in DATA_CACHE:
key = item[0]
tmp[key] = None
tmp['event'] = event_type
tmp['valid'] = 'no'
for timestamp in bad_open_type_events:
tmp['datetime'] = DateTimeUtil.utc_timestamp_to_str(
timestamp, '%Y/%m/%d %H:%M:%S')
attrs.append(OrderedDict(tmp))
return attrs
def prepare_derivation(con, events: list, intervals_before: list,
intervals_after: list, table_name: str,
precision: int, derivation_attr_name: str,
derivation_index: int):
for i in range(0, len(events)):
event = events[i]
no_event_shift = event['no_event_time_shift']
start = event['e_start']['timestamp']
open_value = event['measured'][derivation_attr_name][0]
no_event_open_value = event['no_event_values'][derivation_index]
if no_event_open_value is None:
t = DateTimeUtil.utc_timestamp_to_str(start + no_event_shift,
'%Y-%m-%d %H:%M:%S')
logging.warning('no_event value is None: %s' % t)
# derivacia pred otvorenim okna
# generovanie derivacii medzi hodnou otvorenia okna a hodnotou niekde
# v minulosti, ktora je posunuta o zadany interval dozadu
for interval in intervals_before:
value_time = start - interval
value = Storage.one_row(con, table_name, derivation_attr_name,
value_time)
derivation = None
if value is not None and value[0] is not None:
derivation = round(
(open_value - float(value[0])) / interval, precision)
event['derivation']['before'].append(derivation)
# derivacia po otvoreni okna
# generovanie derviacii medzi hodnotou otvorenia okna a hodnotou niekde,
# v buducnosti, ktora je posunuta o zadany interval dopredu
for interval in intervals_after:
value_time = start + interval
value = Storage.one_row(con, table_name, derivation_attr_name,
value_time)
derivation = None
if value is not None and value[0] is not None:
derivation = round(
(float(value[0]) - open_value) / interval, precision)
event['derivation']['after'].append(derivation)
# derivacia pred no_event
# generovanie derivacii medzi hodnou otvorenia okna a hodnotou niekde
# v minulostia, ktora je posunuta o zadany interval dozadu
# tento cas je posunuty este aj o posun danej udalosti
for interval in intervals_before:
value_time = start + no_event_shift - interval
value = Storage.one_row(con, table_name, derivation_attr_name,
value_time)
derivation = None
if value is not None and value[
0] is not None and no_event_open_value is not None:
derivation = round(
(float(no_event_open_value) - float(value[0])) /
interval, precision)
else:
event['valid_event'] = False
event['derivation']['no_event_before'].append(derivation)
# derivacia pred po no_event
# generovanie derivacii medzi hodnou otvorenia okna a hodnotou niekde
# v minulostia, ktora je posunuta o zadany interval dozadu
# tento cas je posunuty este aj o posun danej udalosti
for interval in intervals_after:
value_time = start + no_event_shift + interval
value = Storage.one_row(con, table_name, derivation_attr_name,
value_time)
derivation = None
if value is not None and value[
0] is not None and no_event_open_value is not None:
derivation = round(
(float(value[0]) - float(no_event_open_value)) /
interval, precision)
else:
event['valid_event'] = False
event['derivation']['no_event_after'].append(derivation)
event['derivation']['intervals_before'] = intervals_before
event['derivation']['intervals_after'] = intervals_after
event['derivation']['intervals_no_event_before'] = intervals_before
event['derivation']['intervals_no_event_after'] = intervals_after
return events