class RedisDataSender(object):
def __init__(self, data_stream_object, read_frequency_hz=5):
self.data_stream_object = data_stream_object
self.read_frequency_hz = read_frequency_hz
self.data_channels = self.data_stream_object.return_data_channels()
# initialize redis connection
redis_instance = Redis(host="redis_host", port="6379")
# initialize redis timeseries client connection
self.rts = Client(conn=redis_instance)
for data_channel in self.data_channels:
self.rts.create(data_channel)
def grab_serial_data(self):
while True:
# grab data tuple from line
tup = self.data_stream_object.read_line()
# walk through all the data channels
for (index, data_channel) in enumerate(self.data_channels):
# pass float into database under correct name
self.send_to_redis_timeseries(tup[index], data_channel)
time.sleep(1 / self.read_frequency_hz) # should operate
def send_to_redis_timeseries(self, flt, data_channel):
self.rts.add(data_channel, "*", flt)
def testPool(self):
redis = Redis(port=port)
client = RedisTimeSeries(conn=redis, port=666)
name = 'test'
client.create(name)
self.assertEqual(None, client.get(name))
client.add(name, 2, 3)
self.assertEqual(2, client.get(name)[0])
info = client.info(name)
self.assertEqual(1, info.total_samples)
class RedisClient:
"""Constants"""
TS_STORE_KEY = "nethive"
__instance = None
@staticmethod
def getInstance():
""" Static access method. """
if RedisClient.__instance == None:
RedisClient()
return RedisClient.__instance
def __init__(self):
""" Virtually private constructor. """
if RedisClient.__instance != None:
# print("RedisClient is a singleton!")
pass
else:
self.__ts_client = Client() # timeseries redis client
self.__redis_client = redis.Redis() # general redis client
# try:
# self.__ts_client.create(self.TS_STORE_KEY)
# except Exception as e:
# pass
RedisClient.__instance = self
# Timeseries Query
def ts_insert_http_bundle(self, store_key, package_id, timestamp, value,
label):
self.__ts_client.create(store_key, labels={'type': 'http'})
return self.__ts_client.add(package_id, timestamp, value, labels=label)
def ts_get_http_bundles(self, start_time, end_time):
return self.__ts_client.mrange(start_time,
end_time,
filters=['type=http'],
with_labels=True)
# return self.__ts_client.info(key)
# return self.__ts_client.mrange(start_time, end_time)
# id = self.__ts_client.range(key, start_time, end_time)
def ts_expire_http_bundle(self, package_id):
self.__ts_client.alter(package_id, labels={"type": "expired"})
key = "{}:{}".format(self.TS_STORE_KEY, package_id)
return self.__ts_client.delete(key)
# End of Timeseries Query
# Redis Query
def store_http_request(self, key, value):
return self.__redis_client.hmset(key, value)
def get_http_request(self, key):
# from_redis = self.__redis_client.hgetall(key)
# self.__redis_client.delete(key)
return self.__redis_client.hgetall(key)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--redis-server",
help="redis server address",
default="localhost")
parser.add_argument("--redis-port",
help="redis server port",
default=6379,
type=int)
args = parser.parse_args()
print("""
Starting with:
redis server: {}
redis port: {}
""".format(args.redis_server, args.redis_port))
rts = RedisTimeSeries(port=args.redis_port, host=args.redis_server)
try:
rts.create('temperature',
retentionSecs=60 * 24,
labels={'sensorId': '2'})
except Exception as e:
# will except if key already exists (i.e. on restart)
print(str(e))
variance = 0
t = 0
while True:
# add with current timestamp
print(".", end="")
variance += (random.random() - 0.5) / 10.0
t += 1
value = math.cos(t / 100) + variance
rts.add('temperature', '*', value)
sleep(0.1)
class RedisTimeSeriesCommon(object):
"""
Wrapper class for accessing RedisTimeSeries.
"""
def __init__(
self,
config: ConfigContextCommon,
name: str = "",
thread: int = 0,
transitionms: int = 0,
retentionms: int = 0,
) -> None:
"""
Args:
config: A config object.
"""
logname = Path(__file__).stem
self._logger = logging.getLogger(f"{config.PACKAGE_NAME}.{logname}")
redis_host = os.environ.get("REDISHOST", "localhost")
redis_port = int(os.environ.get("REDISPORT", 6379))
self._rts = RedisTimeSeries(host=redis_host, port=redis_port)
self._name = name or getattr(config, "name", "A")
self._thread = thread or getattr(config, "thread", 0)
self._transitionms = transitionms or getattr(config, "transitionms",
100)
self._retentionms = retentionms or getattr(config, "retentionms",
7 * 24 * 60 * 60 * 1000)
self._previous_value = 0
def create(
self,
name: str = "",
thread: int = 0,
transitionms: int = 0,
retentionms: int = 0,
) -> None:
if name:
self._name = name
if thread:
self._thread = thread
if transitionms:
self._transitionms = transitionms
if retentionms:
self._retentionms = retentionms
key = f"ts:{self._name}.T:{self._thread:03d}"
labeld = {"ts": self._name, "T": self._thread}
self._rts.create(key, retention_msecs=self._retentionms, labels=labeld)
def delete(self, name: str = "", thread: int = 0) -> None:
key = f"ts:{name or self._name}.T:{thread or self._thread:03d}"
self._rts.delete(key)
# slots are created dynamically and every now and then we want to delete
def delete_slot(self, name: str = "", slot: int = 0) -> None:
key = f"ts:{name or self._name}.S:{slot:03d}"
self._rts.delete(key)
def _add_value(
self,
key: str,
timestampms: Union[int, str],
value: int,
labeld: Mapping[str, Any],
) -> int:
i = 0
while True:
try:
timestampms_return = self._rts.add(
key,
timestampms,
value,
retention_msecs=self._retentionms,
labels=labeld,
)
return timestampms_return # type: ignore
except ResponseError: # too quick, delay a bit if using server timestamp
if i < 5 and timestampms == "*":
i += 1
time.sleep(0.001)
else:
raise
def add_value(self,
value: int = 0,
name: str = "",
thread: int = 0) -> int:
key = f"ts:{name or self._name}.T:{thread or self._thread:03d}"
labeld = {"ts": name or self._name, "T": thread or self._thread}
if self._transitionms and value != self._previous_value:
timestampms_return = self._add_value(key, "*",
self._previous_value, labeld)
time.sleep(self._transitionms / 1000)
self._add_value(key, "*", value, labeld)
self._previous_value = value
return timestampms_return
else:
return self._add_value(key, "*", value, labeld)
def add_slot_values(self,
values: Sequence[int] = [],
name: str = "") -> int:
if not values:
values = [0]
keybase = f"ts:{name or self._name}.S:"
labeld = {"ts": name or self._name, "S": 0}
timestampms = self._add_value(f"{keybase}000", "*", values[0], labeld)
for i, value in enumerate(values[1:]):
j = i + 1
labeld["S"] = j
self._add_value(f"{keybase}{j:03d}", timestampms, value, labeld)
return timestampms
def get_keytuples_by_names(
self,
names: Sequence[str] = [],
types: Sequence[str] = ["T"]) -> List[Tuple[str, int]]:
namelist = (",").join(names or [self._name])
filters = [f"ts=({namelist})"]
keys = self._rts.queryindex(filters)
keytuples = []
for key in keys:
eles = key.split(".")
_, name = eles[0].split(":") # ("ts", <name>)
mytype, value = eles[1].split(":") # ("T" or "S", <str number>)
keytuple = (name, int(value)) # (<name>, <int>)
if mytype in types:
keytuples.append(keytuple)
return keytuples
def get_threads_by_name(self, name: str = "") -> Tuple[int, ...]:
keytuples = self.get_keytuples_by_names([name or self._name],
types=["T"])
names, threads = zip(*keytuples)
return threads # discard names
def get_slots_by_name(self, name: str = "") -> Tuple[int, ...]:
keytuples = self.get_keytuples_by_names([name or self._name],
types=["S"])
names, slots = zip(*keytuples)
return slots # discard names
def _get_dataframe(self, key: str, timestampms: int) -> pd.DataFrame:
datapointts = self._rts.range(key, timestampms, -1)
if not datapointts:
return pd.DataFrame()
dts, values = zip(*datapointts)
datapointdf = pd.DataFrame({
"dt": dts,
key: [float(v) for v in values]
})
datapointdf["dt"] = pd.to_datetime(datapointdf.dt, unit="ms")
return datapointdf.set_index("dt")
def get_dataframe(self,
name: str = "",
thread: int = 0,
timestampms: int = 0) -> pd.DataFrame:
key = f"ts:{name or self._name}.T:{thread or self._thread:03d}"
return self._get_dataframe(key, timestampms)
def get_slot_dataframe(self,
name: str = "",
slot: int = 0,
timestampms: int = 0) -> pd.DataFrame:
key = f"ts:{name or self._name}.S:{slot:03d}"
return self._get_dataframe(key, timestampms)
import pandas as pd
from redistimeseries.client import Client
from datetime import datetime
key = 'test'
rts = Client()
rts.redis.delete(key)
rts.create(key, labels={'source': key})
df = pd.read_csv('samples.csv')
for _, row in df.iterrows():
d = datetime.strptime(row['ds'], '%Y-%m-%d')
millisec = round(d.timestamp()*1000)
rts.add(key, millisec, row['y'])
from redistimeseries.client import Client
from redis.exceptions import ResponseError
from random import randint
import argparse
from time import time, sleep
parser = argparse.ArgumentParser()
parser.add_argument("key", help="Time Series Key", type=str)
parser.add_argument("retention",
help="Time series retention periods in ms",
type=int)
args = parser.parse_args()
rts = Client(host='localhost', port=6379)
try:
rts.create(args.key, retention_msecs=args.retention)
except ResponseError as e:
s = str(e)
print(s)
pass
if __name__ == "__main__":
while (True):
v = randint(0, 10)
t = rts.add(args.key, value=v, timestamp=int(time()))
print('INSERT: ({},{}) AT {}, SUCCESS'.format(args.key, v, t))
sleep(60)
####### This global code is executed once at Lambda/Greengrass startup #######
##############################################################################
total_events = 0
# load_events = 0
load_velocity_sec = 1
timer_for_average = time.time()
while True:
msg = r.blpop("extract_queue")
# Convert CAN Bus message to JSON
converted_message = transforms.obd2_convert(msg[1])
# Update moving averages if message matches
redis_ts = int(float(converted_message["Timestamp"]) * 1000)
if converted_message["Pid"] == "VehicleSpeed":
rts.add("vehicle_speed", redis_ts, converted_message["Value"])
elif converted_message["Pid"] == "ThrottlePosition":
rts.add("throttle_position", redis_ts, converted_message["Value"])
elif converted_message["Pid"] == "AmbientAirTemperature":
rts.add("ambient_air_temperature", redis_ts,
converted_message["Value"])
# Create time series messages on average duration every 30 seconds
cur_time = time.time()
if (cur_time - timer_for_average) > 30:
from_time = int(timer_for_average * 1000)
to_time = int(cur_time * 1000)
msg = {
"VehicleSpeed": {
"Avg":
round(ts_val(rts, "vehicle_speed", from_time, to_time, "avg"),
2),
class Patient:
def __init__(self, name):
self.name = name
self.id = "autogenerateID"
self.rts = Client()
self.delete_previous()
self.create_all_series()
def delete_previous(self):
"""delete any previously same name timeseries."""
self.rts.delete("blood_min")
self.rts.delete("blood_max")
self.rts.delete("sp02")
self.rts.delete("pulse")
print("All previous timeseries deleted")
def create_all_series(self):
"""Create all the time series that will contain all of the patients' data
"""
self.rts.create('blood_min', labels={'Time': 'Series'})
self.rts.create('blood_max', labels={'Time': 'Series'})
self.rts.create('sp02', labels={'Time': 'Series'})
self.rts.create('pulse', labels={'Time': 'Series'})
print("Timeseries created")
def add_blood_samples(self, blood_reading):
"""Convert a standard blood reading into a BloodPressureReading object and
add it to both timeseries.
"""
blood_reading = BloodPressureReading(blood_reading[0], blood_reading[1])
self.rts.add("blood_min", blood_reading.time, blood_reading.min_bp)
# print(rts.get("blood_min"))
self.rts.add("blood_max", blood_reading.time, blood_reading.max_bp)
# print(rts.get("blood_max"))
def add_sp02_samples(self, sp02_reading):
"""Convert a standard SPO2 reading into a SP02Reading and add to its timeseries"""
sp02_reading = SP02Reading(sp02_reading)
self.rts.add("sp02", sp02_reading.time, sp02_reading.oxygen)
def add_pulse_samples(self, pulse_reading):
"""Convert a standard pulse reading into a PulseReading and add to its timeseries"""
pulse_reading = PulseReading(pulse_reading)
self.rts.add("pulse", pulse_reading.time, pulse_reading.heartbeat)
def add_temp_samples(self, temp_reading):
"""Convert a standard temperature (Celsius) into a TemperatureReading and add it to its timeseries"""
temp_reading = TempReading(temp_reading)
self.rts.add("temp", temp_reading.time, temp_reading.temp)
def receive_vitals(self, bp_reading, sp_reading, pulse_reading, temp_reading):
"""Receive vitals from a push operation and add to the timeseries"""
self.add_blood_samples(bp_reading)
self.add_sp02_samples(sp_reading)
self.add_pulse_samples(pulse_reading)
self.add_temp_samples(temp_reading)
print("vitals updated")
#p = Patient("Boris")
#p.receive_vitals([120, 60], 98, 100)
# time.sleep(2)
# p.receive_vitals([130, 90], 80, 170)
#from here on, experiments
# def get_all_ts(timeseries_name):
# full_timeseries = rts.getrange(timeseries_name, 0, -1)
# return full_timeseries
#
# print(get_all_ts("pulse"))
# pulse_sample_boris= pulseReading(98)
# r.hset("boris2", rts.create("borisblood", pulse_sample_boris.time, pulse_sample_boris.heartbeat))
# print(r.hget("boris2"))