def write_data(conn, pk, series):
hash_key = _timeseries_hash_key(pk=pk) # key to the hash of timeseries
set_key = _timeseries_block_starts(pk=pk) # key to set of years with values
ts = read_ts(conn, pk) # read ts attributes
# generator for all years in the given period
years = range(series.index[0].year, series.index[-1].year+1)
with locked(conn, hash_key): # lock actions on timeseries
# get index of first value in the first year
start_index = _get_start_index(series.index[0], ts.freq, ts.tz)
for year in years: # iterate over the years
vkey = _timeseries_block(pk, year)# key to the values of the year
leap = _is_leap(year) # is leapyear?
values = series[str(year)].values # view on the values of the year
if not conn.sismember(set_key, year):
# no data for this year, write a whole year
arr = _initial_year_block(ts.freq, leap)
# write from first index with a value the values to this block
arr[start_index:start_index+len(values)] = values
# start is now zero
start_index=0
else:
arr = values
# format of values: len(arr) doubles
fmt = 'd' * len(arr)
# write to redis a bitmap of the array, each value needs 8 byte
conn.setrange(vkey, start_index*8, struct.pack(fmt, *arr))
# add year to set
conn.sadd(set_key, year)
# for followin years: start will always be zero
start_index=0
def delete_ts(conn, pk):
hash_key = _timeseries_hash_key(pk=pk)
set_key = _timeseries_block_starts(pk=pk)
with locked(conn, hash_key):
years = conn.smembers(set_key)
block_keys = [
_timeseries_block(pk=pk, year=y) for y in map(int, years)
]
conn.delete(hash_key, set_key, *block_keys)
def read_ts(conn, pk):
hash_key = _timeseries_hash_key(pk)
with locked(conn, hash_key):
exists = conn.exists(hash_key)
if not exists:
raise RuntimeWarning('no timeseries with pk {}'.format(pk))
result = conn.hgetall(hash_key)
pk = int(result.get(_HF_PK))
name = result.get(_HF_NAME).decode()
freq = result.get(_HF_FREQ).decode()
tz = result.get(_HF_TZ).decode()
ts = ts_struct(pk, name, freq, tz)
return ts
def read_data(conn, pk, start, end):
hash_key = _timeseries_hash_key(pk=pk)
ts = read_ts(conn, pk)
index = pd.date_range( # index with all dates of the
start=_trans(start), # given period from start to
end=_trans(end), # end
freq=ts.freq,
tz='UTC',
)
index = index.tz_convert(ts.tz) # to tz of timeseries
years = range(index[0].year, index[-1].year+1)
# amount of values to be read. Needed for calculation of end indizes of the
# years
rest_vals = len(index)
# initialize an array with NaNs and datatype double
data = np.nan * np.empty(rest_vals, dtype='<f8')
with locked(conn, hash_key):
start_index = _get_start_index(index[0], ts.freq, ts.tz)
# index for inserting in data array
data_index = 0
for year in years:
vkey = _timeseries_block(pk, year)
steps = _steps_in_year(year, ts.freq)
# do not write a whole block if steps > rest_values
end_index = steps if rest_vals > steps else rest_vals
raw_data = conn.getrange(vkey, start_index*8, end_index*8-1)
#convert bitmap to array of double
small_arr = np.fromstring(raw_data, dtype='<f8')
size = len(small_arr)
# sets the len(small_arr) values beginning at data_index to the
# values of small_arr
data[data_index: data_index+size] = small_arr
data_index += size
rest_vals -= size
start_index = 0
return pd.Series(data=data, index=index)
