def row(self, column_name, time): res = Storage.one_row(self.con, self.table_name, column_name, time) if res is None or res[0] is None: t = DateTimeUtil.utc_timestamp_to_str(time, '%Y/%m/%d %H:%M:%S') raise ValueError('empty value at %s' % t) return float(res[0])
def row(self, column_name, time): if column_name not in self.cache: self.cache[column_name] = {} value = None if time in self.cache[column_name]: value = self.cache[column_name][time] else: res = Storage.one_row(self.con, self.table_name, column_name, time) if res is not None and res[0] is not None: self.cache[column_name][time] = float(res[0]) value = float(res[0]) if value is None: t = DateTimeUtil.utc_timestamp_to_str(time, '%Y/%m/%d %H:%M:%S') raise ValueError('empty value at %s' % t) return value
def testing_data(con, table_name, start, end, write_each, func, row_selector, interval_selector, event_type, log_every_hour=3): """Generation of testing data. :param con: :param table_name: table name :param start: interval from which testing data is generated :param end: interval to which testing data is generated :param write_each: :param func: :param row_selector: :param interval_selector: :return: """ attrs = [] bad_open_type_events = [] global DATA_CACHE for t in range(start, end): if t % (log_every_hour * 3600) == 0: logging.debug(DateTimeUtil.utc_timestamp_to_str(t)) act_row = None if act_row is None: previous_row = Storage.one_row(con, table_name, 'open_close', t - 1) else: previous_row = act_row act_row = Storage.one_row(con, table_name, 'open_close', t) if event_type not in ['open', 'close']: raise ValueError('event type must be: open or close') if previous_row is None or act_row is None: continue open_state = 'nothing' if event_type == 'open' and previous_row[0] == 0 and act_row[ 0] == 1: open_state = event_type elif event_type == 'close' and previous_row[0] == 1 and act_row[ 0] == 0: open_state = event_type if open_state == 'nothing': if t % write_each != 0: continue try: DATA_CACHE = func(con, table_name, t, row_selector, interval_selector) except Exception as e: # logging.error(str(e)) if open_state in ['open', 'close']: bad_open_type_events.append(t) continue time = DateTimeUtil.utc_timestamp_to_str(t, '%Y/%m/%d %H:%M:%S') DATA_CACHE.insert(0, ('datetime', time)) DATA_CACHE.insert(1, ('event', open_state)) DATA_CACHE.append(('valid', 'yes')) attrs.append(OrderedDict(DATA_CACHE)) if DATA_CACHE is None: logging.warning('any {0} events can be skipped'.format(event_type)) else: tmp = {} for item in DATA_CACHE: key = item[0] tmp[key] = None tmp['event'] = event_type tmp['valid'] = 'no' for timestamp in bad_open_type_events: tmp['datetime'] = DateTimeUtil.utc_timestamp_to_str( timestamp, '%Y/%m/%d %H:%M:%S') attrs.append(OrderedDict(tmp)) return attrs
def prepare_derivation(con, events: list, intervals_before: list, intervals_after: list, table_name: str, precision: int, derivation_attr_name: str, derivation_index: int): for i in range(0, len(events)): event = events[i] no_event_shift = event['no_event_time_shift'] start = event['e_start']['timestamp'] open_value = event['measured'][derivation_attr_name][0] no_event_open_value = event['no_event_values'][derivation_index] if no_event_open_value is None: t = DateTimeUtil.utc_timestamp_to_str(start + no_event_shift, '%Y-%m-%d %H:%M:%S') logging.warning('no_event value is None: %s' % t) # derivacia pred otvorenim okna # generovanie derivacii medzi hodnou otvorenia okna a hodnotou niekde # v minulosti, ktora je posunuta o zadany interval dozadu for interval in intervals_before: value_time = start - interval value = Storage.one_row(con, table_name, derivation_attr_name, value_time) derivation = None if value is not None and value[0] is not None: derivation = round( (open_value - float(value[0])) / interval, precision) event['derivation']['before'].append(derivation) # derivacia po otvoreni okna # generovanie derviacii medzi hodnotou otvorenia okna a hodnotou niekde, # v buducnosti, ktora je posunuta o zadany interval dopredu for interval in intervals_after: value_time = start + interval value = Storage.one_row(con, table_name, derivation_attr_name, value_time) derivation = None if value is not None and value[0] is not None: derivation = round( (float(value[0]) - open_value) / interval, precision) event['derivation']['after'].append(derivation) # derivacia pred no_event # generovanie derivacii medzi hodnou otvorenia okna a hodnotou niekde # v minulostia, ktora je posunuta o zadany interval dozadu # tento cas je posunuty este aj o posun danej udalosti for interval in intervals_before: value_time = start + no_event_shift - interval value = Storage.one_row(con, table_name, derivation_attr_name, value_time) derivation = None if value is not None and value[ 0] is not None and no_event_open_value is not None: derivation = round( (float(no_event_open_value) - float(value[0])) / interval, precision) else: event['valid_event'] = False event['derivation']['no_event_before'].append(derivation) # derivacia pred po no_event # generovanie derivacii medzi hodnou otvorenia okna a hodnotou niekde # v minulostia, ktora je posunuta o zadany interval dozadu # tento cas je posunuty este aj o posun danej udalosti for interval in intervals_after: value_time = start + no_event_shift + interval value = Storage.one_row(con, table_name, derivation_attr_name, value_time) derivation = None if value is not None and value[ 0] is not None and no_event_open_value is not None: derivation = round( (float(value[0]) - float(no_event_open_value)) / interval, precision) else: event['valid_event'] = False event['derivation']['no_event_after'].append(derivation) event['derivation']['intervals_before'] = intervals_before event['derivation']['intervals_after'] = intervals_after event['derivation']['intervals_no_event_before'] = intervals_before event['derivation']['intervals_no_event_after'] = intervals_after return events