def _n_from_marker_numbers(self, tytup, item, topic, date_time, n, throw_if_data_lost): nums = (tytup[1] * n)() dts = (_DateTimeStamp * n)() g = _long_() _lib_call("TOSDB_GetN" + tytup[0] + "sFromMarker", self._name, item.encode("ascii"), topic.encode("ascii"), nums, n, dts if date_time else _PTR_(_DateTimeStamp)(), _pointer(g), arg_types=(_str_, _str_, _str_, _PTR_(tytup[1]), _uint32_, _PTR_(_DateTimeStamp), _PTR_(_long_))) g = g.value if g == 0: return None elif g < 0: if throw_if_data_lost: raise TOSDB_DataError("data lost behind the 'marker'") else: g *= -1 return list(zip(nums[:g], _map_dt(dts[:g])) if date_time else nums[:g])
def _n_from_marker_strings(self, item, topic, date_time, n, throw_if_data_lost, data_str_max): strs = _gen_str_buffers(data_str_max + 1, n) pstrs = _gen_str_buffers_ptrs(strs) dts = (_DateTimeStamp * n)() g = _long_() _lib_call("TOSDB_GetNStringsFromMarker", self._name, item.encode("ascii"), topic.encode("ascii"), pstrs, n, data_str_max + 1, dts if date_time else _PTR_(_DateTimeStamp)(), _pointer(g), arg_types=(_str_, _str_, _str_, _ppchar_, _uint32_, _uint32_, _PTR_(_DateTimeStamp), _PTR_(_long_))) g = g.value if g == 0: return None elif g < 0: if throw_if_data_lost: raise TOSDB_DataError("data lost behind the 'marker'") else: g *= -1 return list( _zip_cstr_dt(pstrs[:g], dts[:g] ) if date_time else _map_cstr(pstrs[:g]))
def stream_snapshot_from_marker(self, item, topic, date_time=False, beg=0, margin_of_safety=100, throw_if_data_lost=True, data_str_max=STR_DATA_SZ): item = item.upper() topic = topic.upper() if date_time and not self._date_time: raise TOSDB_DateTimeError("date_time not available for this block") self._valid_item(item) self._valid_topic(topic) if beg < 0: beg += self._block_size if beg < 0 or beg >= self._block_size: raise TOSDB_IndexError("invalid 'beg' index value") if margin_of_safety < MIN_MARGIN_OF_SAFETY: raise TOSDB_ValueError("margin_of_safety < MIN_MARGIN_OF_SAFETY") is_dirty = _uint_() _lib_call("TOSDB_IsMarkerDirty", self._name, item.encode("ascii"), topic.encode("ascii"), _pointer(is_dirty), arg_types=(_str_, _str_, _str_, _PTR_(_uint_))) if is_dirty and throw_if_data_lost: raise TOSDB_DataError("marker is already dirty") mpos = _longlong_() _lib_call("TOSDB_GetMarkerPosition", self._name, item.encode("ascii"), topic.encode("ascii"), _pointer(mpos), arg_types=(_str_, _str_, _str_, _PTR_(_longlong_))) cur_sz = mpos.value - beg + 1 if cur_sz < 0: return None safe_sz = cur_sz + margin_of_safety dtss = (_DateTimeStamp * safe_sz)() tbits = type_bits(topic) tytup = _type_switch(tbits) get_size = _long_() if tytup[0] == "String": strs = [_BUF_(data_str_max + 1) for _ in range(safe_sz)] strs_array = (_pchar_ * safe_sz)(*[_cast(s, _pchar_) for s in strs]) _lib_call("TOSDB_GetStreamSnapshotStringsFromMarker", self._name, item.encode("ascii"), topic.encode("ascii"), strs_array, safe_sz, data_str_max + 1, dtss if date_time else _PTR_(_DateTimeStamp)(), beg, _pointer(get_size), arg_types=(_str_, _str_, _str_, _ppchar_, _uint32_, _uint32_, _PTR_(_DateTimeStamp), _long_, _PTR_(_long_))) get_size = get_size.value if get_size == 0: return None elif get_size < 0: if throw_if_data_lost: raise TOSDB_DataError("data lost behind the 'marker'") else: get_size *= -1 if date_time: adj_dts = [TOSDB_DateTime(x) for x in dtss[:get_size]] return [ sd for sd in zip(map(_cast_cstr, strs_array[:get_size]), adj_dts) ] else: return [_cast_cstr(s) for s in strs_array[:get_size]] else: num_array = (tytup[1] * safe_sz)() _lib_call("TOSDB_GetStreamSnapshot" + tytup[0] + "sFromMarker", self._name, item.encode("ascii"), topic.encode("ascii"), num_array, safe_sz, dtss if date_time else _PTR_(_DateTimeStamp)(), beg, _pointer(get_size), arg_types=(_str_, _str_, _str_, _PTR_(tytup[1]), _uint32_, _PTR_(_DateTimeStamp), _long_, _PTR_(_long_))) get_size = get_size.value if get_size == 0: return None elif get_size < 0: if throw_if_data_lost: raise TOSDB_DataError("data lost behind the 'marker'") else: get_size *= -1 if date_time: adj_dts = [TOSDB_DateTime(x) for x in dtss[:get_size]] return [nd for nd in zip(num_array[:get_size], adj_dts)] else: return [n for n in num_array[:get_size]]
def stream_snapshot_from_marker(self, item, topic, date_time=False, beg=0, margin_of_safety=100, throw_if_data_lost=True, data_str_max=STR_DATA_SZ): item = item.upper() topic = topic.upper() if date_time and not self._date_time: raise TOSDB_DateTimeError("date_time not available for this block") self._valid_item(item) self._valid_topic(topic) if beg < 0: beg += self._block_size if beg < 0 or beg >= self._block_size: raise TOSDB_IndexError("invalid 'beg' index value") if margin_of_safety < MIN_MARGIN_OF_SAFETY: raise TOSDB_ValueError("margin_of_safety < MIN_MARGIN_OF_SAFETY") is_dirty = _uint_() _lib_call("TOSDB_IsMarkerDirty", self._name, item.encode("ascii"), topic.encode("ascii"), _pointer(is_dirty), arg_types=(_str_, _str_, _str_, _PTR_(_uint_))) if is_dirty and throw_if_data_lost: raise TOSDB_DataError("marker is already dirty") mpos = _longlong_() _lib_call("TOSDB_GetMarkerPosition", self._name, item.encode("ascii"), topic.encode("ascii"), _pointer(mpos), arg_types=(_str_, _str_, _str_, _PTR_(_longlong_))) cur_sz = mpos.value - beg + 1 if cur_sz < 0: return None safe_sz = cur_sz + margin_of_safety dtss = (_DateTimeStamp * safe_sz)() tbits = type_bits(topic) tytup = _type_switch(tbits) get_size = _long_() if tytup[0] == "String": strs = [ _BUF_( data_str_max +1) for _ in range(safe_sz) ] strs_array = (_pchar_ * safe_sz)(*[_cast(s,_pchar_) for s in strs]) _lib_call("TOSDB_GetStreamSnapshotStringsFromMarker", self._name, item.encode("ascii"), topic.encode("ascii"), strs_array, safe_sz, data_str_max + 1, dtss if date_time else _PTR_(_DateTimeStamp)(), beg, _pointer(get_size), arg_types=(_str_, _str_, _str_, _ppchar_, _uint32_, _uint32_, _PTR_(_DateTimeStamp), _long_, _PTR_(_long_))) get_size = get_size.value if get_size == 0: return None elif get_size < 0: if throw_if_data_lost: raise TOSDB_DataError("data lost behind the 'marker'") else: get_size *= -1 if date_time: adj_dts = [TOSDB_DateTime(x) for x in dtss[:get_size]] return [sd for sd in zip(map(_cast_cstr, strs_array[:get_size]), adj_dts)] else: return [_cast_cstr(s) for s in strs_array[:get_size]] else: num_array = (tytup[1] * safe_sz)() _lib_call("TOSDB_GetStreamSnapshot" + tytup[0] + "sFromMarker", self._name, item.encode("ascii"), topic.encode("ascii"), num_array, safe_sz, dtss if date_time else _PTR_(_DateTimeStamp)(), beg, _pointer(get_size), arg_types=(_str_, _str_, _str_, _PTR_(tytup[1]), _uint32_, _PTR_(_DateTimeStamp), _long_, _PTR_(_long_))) get_size = get_size.value if get_size == 0: return None elif get_size < 0: if throw_if_data_lost: raise TOSDB_DataError("data lost behind the 'marker'") else: get_size *= -1 if date_time: adj_dts = [TOSDB_DateTime(x) for x in dtss[:get_size]] return [nd for nd in zip(num_array[:get_size], adj_dts)] else: return [n for n in num_array[:get_size]]
def stream_snapshot_from_marker( self, item, topic, date_time = False, beg = 0, margin_of_safety = 100, throw_if_data_lost = True, data_str_max = STR_DATA_SZ ): """ Return multiple data-points(a snapshot) from the data-stream, ending where the last call began It's likely the stream will grow between consecutive calls. This call guarantees to pick up where the last get(), stream_snapshot(), or stream_snapshot_from_marker() call ended (under a few assumptions, see below). Internally the stream maintains a 'marker' that tracks the position of the last value pulled; the act of retreiving data and moving the marker can be thought of as a single, 'atomic' operation. There are three states to be aware of: 1) a 'beg' value that is greater than the marker (even if beg = 0) 2) a marker that moves through the entire stream and hits the bound 3) passing a buffer that is too small for the whole range State (1) can be caused by passing in a beginning index that is past the current marker, or by passing in 0 when the marker has yet to move. 'None' will be returned. State (2) occurs when the marker doesn't get reset before it hits the bound (block_size); as the oldest data is popped of the back of the stream it is lost (the marker can't grow past the end of the stream). State (3) occurs when an inadequately small buffer is used. The call handles buffer sizing for you by calling down to get the marker index, adjusting by 'beg' and 'margin_of_safety'. The latter helps assure the marker doesn't outgrow the buffer by the time the low-level retrieval operation completes. The default value indicates that over 100 push operations would have to take place during this call(highly unlikely). In either case (state (2) or (3)) if throw_if_data_lost is True a TOSDB_DataError will be thrown, otherwise the available data will be returned as normal. item: any item string in the block topic: any topic string in the block date_time: (True/False) attempt to retrieve a TOSDB_DateTime object beg: index of most recent data-point ( beginning of the snapshot ) margin_of_safety: (True/False) error margin for async stream growth throw_if_data_loss: (True/False) how to handle error states (see above) data_str_max: the maximum length of string data returned if beg > internal marker value: returns -> None if date_time is True: returns-> list of 2tuple else: returns -> list """ item = item.upper() topic = topic.upper() if date_time and not self._date_time: raise TOSDB_DateTimeError("date_time not available for this block") self._valid_item(item) self._valid_topic(topic) if beg < 0: beg += self._block_size if beg < 0 or beg >= self._block_size: raise TOSDB_IndexError("invalid 'beg' index value") if margin_of_safety < MIN_MARGIN_OF_SAFETY: raise TOSDB_ValueError("margin_of_safety < MIN_MARGIN_OF_SAFETY") is_dirty = _uint_() err = _lib_call( "TOSDB_IsMarkerDirty", self._name, item.encode("ascii"), topic.encode("ascii"), _pointer(is_dirty), arg_list = [ _str_, _str_, _str_, _PTR_(_uint_) ] ) if err: raise TOSDB_CLibError( "error value [ "+ str(err) + " ] " + "returned from library call", "TOSDB_IsMarkerDirty" ) if is_dirty and throw_if_data_lost: raise TOSDB_DataError("marker is already dirty") mpos = _longlong_() err2 = _lib_call( "TOSDB_GetMarkerPosition", self._name, item.encode("ascii"), topic.encode("ascii"), _pointer(mpos), arg_list = [ _str_, _str_, _str_, _PTR_(_longlong_) ]) if err2: raise TOSDB_CLibError( "error value [ "+ str(err2) + " ] " + "returned from library call", "TOSDB_GetMarkerPosition" ) cur_sz = mpos.value - beg + 1 if cur_sz < 0: return None safe_sz = cur_sz + margin_of_safety dtss = (_DateTimeStamp * safe_sz)() tbits = type_bits( topic ) tytup = _type_switch( tbits ) get_size = _long_() if tytup[0] == "String": # store char buffers strs = [ _BUF_( data_str_max +1 ) for _ in range(safe_sz) ] # cast char buffers into (char*)[ ] strs_array = (_pchar_ * safe_sz)(*[ cast(s,_pchar_) for s in strs]) err3 = _lib_call( "TOSDB_GetStreamSnapshotStringsFromMarker", self._name, item.encode("ascii"), topic.encode("ascii"), strs_array, safe_sz, data_str_max + 1, dtss if date_time else _PTR_(_DateTimeStamp)(), beg, _pointer( get_size ), arg_list = [ _str_, _str_, _str_, _ppchar_, _ulong_, _ulong_, _PTR_(_DateTimeStamp), _long_, _PTR_(_long_) ] ) if err3: raise TOSDB_CLibError( "error value [ " + str(err3) + " ] returned from library call", "TOSDB_GetStreamSnapshotStrings" \ + "FromMarker") get_size = get_size.value if get_size == 0: return None elif get_size < 0: if throw_if_data_lost: raise TOSDB_DataError("data lost behind the 'marker'") else: get_size *= -1 if date_time: adj_dts = [ TOSDB_DateTime( x ) for x in dtss[:get_size] ] return [ _ for _ in \ zip( map( lambda x : cast(x, _str_).value.decode(), strs_array[:get_size] ), adj_dts ) ] else: return [ cast(ptr,_str_).value.decode() for ptr in strs_array[:get_size] ] else: num_array = (tytup[1] * safe_sz)() err3 = _lib_call( "TOSDB_GetStreamSnapshot" \ + tytup[0] + "sFromMarker" , self._name, item.encode("ascii"), topic.encode("ascii"), num_array, safe_sz, dtss if date_time else _PTR_(_DateTimeStamp)(), beg, _pointer( get_size ), arg_list = [ _str_, _str_, _str_, _PTR_(tytup[1]), _ulong_, _PTR_(_DateTimeStamp), _long_, _PTR_(_long_) ] ) if err3: raise TOSDB_CLibError( "error value of [ " + str(err3) + " ] returned from library call", "TOSDB_GetStreamSnapshot" \ + tytup[0] + "sFromMarker" ) get_size = get_size.value if get_size == 0: return None elif get_size < 0: if throw_if_data_lost: raise TOSDB_DataError("data lost behind the 'marker'") else: get_size *= -1 if date_time: adj_dts = [ TOSDB_DateTime( x ) for x in dtss[:get_size] ] return [ _ for _ in zip( num_array[:get_size], adj_dts ) ] else: return [ _ for _ in num_array[:get_size] ]
def stream_snapshot_from_marker(self, item, topic, date_time=False, beg=0, margin_of_safety=100, throw_if_data_lost=True, data_str_max=STR_DATA_SZ): """ Return multiple data-points(a snapshot) from the data-stream, ending where the last call began It's likely the stream will grow between consecutive calls. This call guarantees to pick up where the last get(), stream_snapshot(), or stream_snapshot_from_marker() call ended (under a few assumptions, see below). Internally the stream maintains a 'marker' that tracks the position of the last value pulled; the act of retreiving data and moving the marker can be thought of as a single, 'atomic' operation. There are three states to be aware of: 1) a 'beg' value that is greater than the marker (even if beg = 0) 2) a marker that moves through the entire stream and hits the bound 3) passing a buffer that is too small for the whole range State (1) can be caused by passing in a beginning index that is past the current marker, or by passing in 0 when the marker has yet to move. 'None' will be returned. State (2) occurs when the marker doesn't get reset before it hits the bound (block_size); as the oldest data is popped of the back of the stream it is lost (the marker can't grow past the end of the stream). State (3) occurs when an inadequately small buffer is used. The call handles buffer sizing for you by calling down to get the marker index, adjusting by 'beg' and 'margin_of_safety'. The latter helps assure the marker doesn't outgrow the buffer by the time the low-level retrieval operation completes. The default value indicates that over 100 push operations would have to take place during this call(highly unlikely). In either case (state (2) or (3)) if throw_if_data_lost is True a TOSDB_DataError will be thrown, otherwise the available data will be returned as normal. item: any item string in the block topic: any topic string in the block date_time: (True/False) attempt to retrieve a TOSDB_DateTime object beg: index of most recent data-point ( beginning of the snapshot ) margin_of_safety: (True/False) error margin for async stream growth throw_if_data_loss: (True/False) how to handle error states (see above) data_str_max: the maximum length of string data returned if beg > internal marker value: returns -> None if date_time is True: returns-> list of 2tuple else: returns -> list """ item = item.upper() topic = topic.upper() if date_time and not self._date_time: raise TOSDB_DateTimeError("date_time not available for this block") self._valid_item(item) self._valid_topic(topic) if beg < 0: beg += self._block_size if beg < 0 or beg >= self._block_size: raise TOSDB_IndexError("invalid 'beg' index value") if margin_of_safety < MIN_MARGIN_OF_SAFETY: raise TOSDB_ValueError("margin_of_safety < MIN_MARGIN_OF_SAFETY") is_dirty = _uint_() err = _lib_call("TOSDB_IsMarkerDirty", self._name, item.encode("ascii"), topic.encode("ascii"), _pointer(is_dirty), arg_list=[_str_, _str_, _str_, _PTR_(_uint_)]) if err: raise TOSDB_CLibError( "error value [ " + str(err) + " ] " + "returned from library call", "TOSDB_IsMarkerDirty") if is_dirty and throw_if_data_lost: raise TOSDB_DataError("marker is already dirty") mpos = _longlong_() err2 = _lib_call("TOSDB_GetMarkerPosition", self._name, item.encode("ascii"), topic.encode("ascii"), _pointer(mpos), arg_list=[_str_, _str_, _str_, _PTR_(_longlong_)]) if err2: raise TOSDB_CLibError( "error value [ " + str(err2) + " ] " + "returned from library call", "TOSDB_GetMarkerPosition") cur_sz = mpos.value - beg + 1 if cur_sz < 0: return None safe_sz = cur_sz + margin_of_safety dtss = (_DateTimeStamp * safe_sz)() tbits = type_bits(topic) tytup = _type_switch(tbits) get_size = _long_() if tytup[0] == "String": # store char buffers strs = [_BUF_(data_str_max + 1) for _ in range(safe_sz)] # cast char buffers into (char*)[ ] strs_array = (_pchar_ * safe_sz)(*[cast(s, _pchar_) for s in strs]) err3 = _lib_call("TOSDB_GetStreamSnapshotStringsFromMarker", self._name, item.encode("ascii"), topic.encode("ascii"), strs_array, safe_sz, data_str_max + 1, dtss if date_time else _PTR_(_DateTimeStamp)(), beg, _pointer(get_size), arg_list=[ _str_, _str_, _str_, _ppchar_, _ulong_, _ulong_, _PTR_(_DateTimeStamp), _long_, _PTR_(_long_) ]) if err3: raise TOSDB_CLibError( "error value [ " + str(err3) + " ] returned from library call", "TOSDB_GetStreamSnapshotStrings" \ + "FromMarker") get_size = get_size.value if get_size == 0: return None elif get_size < 0: if throw_if_data_lost: raise TOSDB_DataError("data lost behind the 'marker'") else: get_size *= -1 if date_time: adj_dts = [TOSDB_DateTime(x) for x in dtss[:get_size]] return [ _ for _ in \ zip( map( lambda x : cast(x, _str_).value.decode(), strs_array[:get_size] ), adj_dts ) ] else: return [ cast(ptr, _str_).value.decode() for ptr in strs_array[:get_size] ] else: num_array = (tytup[1] * safe_sz)() err3 = _lib_call( "TOSDB_GetStreamSnapshot" \ + tytup[0] + "sFromMarker" , self._name, item.encode("ascii"), topic.encode("ascii"), num_array, safe_sz, dtss if date_time else _PTR_(_DateTimeStamp)(), beg, _pointer( get_size ), arg_list = [ _str_, _str_, _str_, _PTR_(tytup[1]), _ulong_, _PTR_(_DateTimeStamp), _long_, _PTR_(_long_) ] ) if err3: raise TOSDB_CLibError( "error value of [ " + str(err3) + " ] returned from library call", "TOSDB_GetStreamSnapshot" \ + tytup[0] + "sFromMarker" ) get_size = get_size.value if get_size == 0: return None elif get_size < 0: if throw_if_data_lost: raise TOSDB_DataError("data lost behind the 'marker'") else: get_size *= -1 if date_time: adj_dts = [TOSDB_DateTime(x) for x in dtss[:get_size]] return [_ for _ in zip(num_array[:get_size], adj_dts)] else: return [_ for _ in num_array[:get_size]]