def f(r):
key = r['key']
value = r['value']
keys = value.keys()
uuid = value.pop(UUID_KEY, None)
idToAck = '{%s}%s' % (r['key'], uuid)
try:
operation = value[OP_KEY]
mappedValue = {}
mappedValue[ORIGINAL_KEY] = key
mappedValue[self.connector.PrimaryKey()] = key.split(':')[1]
mappedValue[UUID_KEY] = uuid
mappedValue[OP_KEY] = operation
if operation == OPERATION_UPDATE_REPLICATE:
for kInHash, kInDB in self.mappings.items():
if kInHash.startswith('_'):
continue
if kInHash not in keys:
msg = 'Could not find %s in hash %s' % (kInHash,
r['key'])
WriteBehindLog(msg)
raise Exception(msg)
mappedValue[kInDB] = value[kInHash]
func([{'value': mappedValue}])
except Exception as e:
WriteBehindLog("Failed writing data to the database, error='%s'" %
str(e))
# lets update the ack stream to failure
if uuid is not None and uuid != '':
execute('XADD', idToAck, '*', 'status', 'failed', 'error',
str(e))
execute('EXPIRE', idToAck, ackExpireSeconds)
return False
return True
def func(r):
WriteBehindLog("CATSF %s" % r)
data = []
data.append([ORIGINAL_KEY, r['key']])
data.append([self.connector.PrimaryKey(), r['key'].split(':')[1]])
if 'value' in r.keys():
value = r['value']
uuid = value.pop(UUID_KEY, None)
op = value[OP_KEY]
data.append([OP_KEY, op])
keys = value.keys()
if uuid is not None:
data.append([UUID_KEY, uuid])
if op == OPERATION_UPDATE_REPLICATE:
for kInHash, kInDB in self.mappings.items():
if kInHash.startswith('_'):
continue
if kInHash not in keys:
msg = 'Could not find %s in hash %s' % (kInHash,
r['key'])
WriteBehindLog(msg)
raise Exception(msg)
data.append([kInDB, value[kInHash]])
execute('xadd', GetStreamName(self.connector.TableName()), '*',
*sum(data, []))
def UnregisterOldVersions(name, version):
WriteBehindLog('Unregistering old versions of %s' % name)
registrations = execute('rg.dumpregistrations')
for registration in registrations:
registrationDict = RegistrationArrToDict(registration, 0)
descStr = registrationDict['desc']
try:
desc = json.loads(descStr)
except Exception as e:
continue
if 'name' in desc.keys() and name in desc['name']:
WriteBehindLog(
'Version auto upgrade is not atomic, make sure to use it when there is not traffic to the database (otherwise you might lose events).',
logLevel='warning')
if 'version' not in desc.keys():
execute('rg.unregister', registrationDict['id'])
WriteBehindLog('Unregistered %s' % registrationDict['id'])
continue
v = desc['version']
if CompareVersions(version, v) > 0:
execute('rg.unregister', registrationDict['id'])
WriteBehindLog('Unregistered %s' % registrationDict['id'])
else:
raise Exception(
'Found a version which is greater or equals current version, aborting.'
)
WriteBehindLog('Unregistered old versions')
def UpdateHash(r):
key = r['key']
value = r['value']
operation = value.pop(OP_KEY, None)
uuid_key = value.pop(UUID_KEY, None)
if operation == OPERATION_DEL_REPLICATE or operation == OPERATION_DEL_NOREPLICATE:
SafeDeleteKey(key)
elif operation == OPERATION_UPDATE_REPLICATE or OPERATION_UPDATE_NOREPLICATE:
# we need to write to temp key and then rename so we will not
# trigger another execution
tempKeyName = 'temp-{%s}' % key
elemets = []
for k, v in value.items():
elemets.append(k)
elemets.append(v)
#if READ_MODE != 'StreamReader':
execute('hset', tempKeyName, *elemets)
execute('rename', tempKeyName, key)
#else:
# execute('xadd', tempKeyName, *elemets)
# execute('rename', tempKeyName, key)
else:
msg = "Unknown operation"
WriteBehindLog(msg)
raise Exception(msg)
def SafeDeleteKey(key):
'''
Deleting a key by first renaming it so we will not trigger another execution
If key does not exists we will get an execution and ignore it
'''
try:
newKey = '__{%s}__' % key
execute('RENAME', key, newKey)
execute('DEL', newKey)
except Exception:
pass
def downsampleStream(x):
''' Drops input frames to match FPS '''
global _mspf, _next_ts
execute('TS.INCRBY', 'camera:0:in_fps', 1, 'RESET',
1) # Store the input fps count
ts, _ = map(int,
str(x['streamId']).split(
'-')) # Extract the timestamp part from the message ID
sample_it = _next_ts <= ts
if sample_it: # Drop frames until the next timestamp is in the present/past
_next_ts = ts + _mspf
return sample_it
def func(data):
idsToAck = []
for d in data:
originalKey = d['value'].pop(ORIGINAL_KEY, None)
uuid = d['value'].pop(UUID_KEY, None)
if uuid is not None and uuid != '':
idsToAck.append('{%s}%s' % (originalKey, uuid))
connector.WriteData(data)
for idToAck in idsToAck:
execute('XADD', idToAck, '*', 'status', 'done')
execute('EXPIRE', idToAck, ackExpireSeconds)
def PythonReaderCallback():
pref = prefix
cursor = '0'
res = execute('scan', cursor, 'COUNT', '10000', 'MATCH', pref)
cursor = res[0]
keys = res[1]
while int(cursor) != 0:
for k in keys:
yield k
res = execute('scan', cursor, 'COUNT', '10000', 'MATCH', pref)
cursor = res[0]
keys = res[1]
for k in keys:
yield k
def PrepareRecord(r):
key = r['key']
value = r['value']
realKey = key.split('{')[1].split('}')[0]
realVal = execute('hgetall', realKey)
realVal = {realVal[i]: realVal[i + 1] for i in range(0, len(realVal), 2)}
realVal.update(value)
# delete temporary key
execute('del', key)
return {'key': realKey, 'value': realVal}
def ValidateHash(r):
key = r['key']
value = r['value']
WriteBehindLog('MANUEL - START ValidateHash the message: ' + str(r))
if value == None:
# key without value consider delete
value = {OP_KEY: OPERATION_DEL_REPLICATE}
r['value'] = value
else:
# make sure its a hash
if not (isinstance(r['value'], dict)):
msg = 'Got a none hash value, key="%s" value="%s"' % (str(
r['key']), str(r['value'] if 'value' in r.keys() else 'None'))
WriteBehindLog(msg)
raise Exception(msg)
if OP_KEY not in value.keys():
value[OP_KEY] = defaultOperation
else:
# we need to delete the operation key for the hash
execute('hdel', key, OP_KEY)
op = value[OP_KEY]
if len(op) == 0:
msg = 'Got no operation'
WriteBehindLog(msg)
raise Exception(msg)
operation = op[0]
if operation not in OPERATIONS:
msg = 'Got unknown operations "%s"' % operation
WriteBehindLog(msg)
raise Exception(msg)
# lets extrac uuid to ack on
uuid = op[1:]
value[UUID_KEY] = uuid if uuid != '' else None
value[OP_KEY] = operation
r['value'] = value
WriteBehindLog('MANUEL - END ReValidateHashceive the message: ' + str(r))
return True
def WriteData(self, data):
for e in data:
pk = e.pop(SIMPLE_HASH_BACKEND_PK)
streamId = e.pop('streamId')
newKey = '%s:%s' % (self.newPefix, pk)
d = [[k, v] for k, v in e.items() if not k.startswith('_')]
res = execute('hset', newKey, *sum(d, []))
if 'ERR' in str(res):
raise Exception(res)
def func(data):
WriteBehindLog('MANUEL - CreateWriteDataFunction data: ' + str(data))
idsToAck = []
for d in data:
originalKey = d['value'].pop(ORIGINAL_KEY, None)
uuid = d['value'].pop(UUID_KEY, None)
if uuid is not None and uuid != '':
idsToAck.append('{%s}%s' % (originalKey, uuid))
connector.WriteData(data)
WriteBehindLog('MANUEL - CreateWriteDataFunction idsToAck: ' +
str(idsToAck))
for idToAck in idsToAck:
if idToAck is not None:
WriteBehindLog(
'MANUEL - CreateWriteDataFunction idsToAck ENTRAAA')
execute('XADD', idToAck, '*', 'status', 'done')
execute('EXPIRE', idToAck, ackExpireSeconds)
def keysOnlyReader():
nonlocal pattern
nonlocal count
nonlocal noScan
nonlocal patternGenerator
if patternGenerator is not None:
pattern, noScan = patternGenerator()
if noScan:
if execute('exists', pattern) == 1:
yield pattern
else:
count = str(count)
cursor, keys = execute('scan', '0', 'MATCH', str(pattern), 'COUNT',
count)
while cursor != '0':
for k in keys:
yield k
cursor, keys = execute('scan', cursor, 'MATCH', str(pattern),
'COUNT', count)
for k in keys:
yield k
def ShouldProcessHash(r):
key = r['key']
value = r['value']
uuid = value[UUID_KEY]
operation = value[OP_KEY]
res = True
if operation == OPERATION_DEL_NOREPLICATE:
# we need to just delete the key but delete it directly will cause
# key unwanted key space notification so we need to rename it first
SafeDeleteKey(key)
res = False
if operation == OPERATION_UPDATE_NOREPLICATE:
res = False
if not res and uuid != '':
# no replication to connector is needed but ack is require
idToAck = '{%s}%s' % (key, uuid)
execute('XADD', idToAck, '*', 'status', 'done')
execute('EXPIRE', idToAck, ackExpireSeconds)
return res
def storeResults(x):
''' Stores the results in Redis Stream and TimeSeries data structures '''
global _mspf, prf
ref_id, people, boxes = x[0], int(x[1]), x[2]
ref_msec = int(str(ref_id).split('-')[0])
# Store the output in its own stream
res_id = execute('XADD', 'camera:0:yolo', 'MAXLEN', '~', 1000, '*', 'ref',
ref_id, 'boxes', boxes, 'people', people)
# Add a sample to the output people and fps timeseries
res_msec = int(str(res_id).split('-')[0])
ts = ref_msec // 1000
execute('TS.ADD', 'camera:0:people', ts, people)
execute('TS.INCRBY', 'camera:0:out_fps', 1, 'RESET', 1)
# Adjust mspf to the moving average duration
total_duration = res_msec - ref_msec
prf.assign('total', total_duration)
avg_duration = prf.get('total')
_mspf = avg_duration * 1.05 # A little extra leg room
# Record profiler steps
for name in prf.names:
current = prf.data[name].current
execute('TS.ADD', 'camera:0:prf_{}'.format(name), ts, current)
prf.add('store')
# Make an arithmophilial homage to Count von Count for storage in the execution log
if people == 0:
return 'Now there are none.'
elif people == 1:
return 'There is one person in the frame!'
elif people == 2:
return 'And now there are are two!'
else:
return 'I counted {} people in the frame! Ah ah ah!'.format(people)
def shardReaderCallback():
res = execute('RG.INFOCLUSTER')
if res == 'no cluster mode':
yield '1'
else:
yield res[1]