class ManagerBase(object):
def __init__(self, filename):
self.storage = DictStore(filename)
if 'vals' not in self.storage:
self.storage['vals'] = dict()
def _clear(self):
self.storage.clear()
self.storage['vals'] = dict()
@property
def vals(self):
"""
Returns a copy of the resources dict
"""
return self.storage['vals']
@update_storage
def add_item(self, vals, item, name=None):
if name is None:
name = item.name
vals[name] = item
@update_storage
def remove_item(self, vals, name):
del vals[name]
class TransactionManager(object):
def __init__(self, filename='transaction_storage.pickle'):
self.dict_store = DictStore(filename)
self.current_id = self.get_current_id()
self.top_hash = self.get_top_hash()
def _clear(self):
self.current_id = 0
self.dict_store.clear()
self.top_hash = None
def get_current_id(self):
try:
return self.get_val("cur_id")
except KeyError:
return 0
def set_val(self, key, val):
self.dict_store[key] = {"val": val}
def get_val(self, key):
return self.dict_store[key]['val']
def set_current_id(self):
self.set_val("cur_id", self.current_id)
def validate_user_signature(self, user, signature):
"""
Validates the user's signature. Raises an exception on failure
:param user: user
:param signature: user signature
"""
def add_transaction(self, data, ts, user, signature, uuid=None, update_hashes=True):
if uuid is None:
uuid = uuid4()
insert_idx = self._get_prev_transaction_idx(ts, uuid)
if insert_idx is None:
insert_idx = 1
else:
insert_idx += 1
self._bubble_up(insert_idx)
self.validate_user_signature(user, signature)
new_item = {
"data": data,
"ts": ts,
"user": user,
"signature": signature,
"uuid": uuid,
}
hash = self.calculate_item_hash(new_item)
new_item['hash'] = hash
self.dict_store[insert_idx] = new_item
ts_list = self.get_by_timestamp(ts)
new_item.pop('hash', None)
ts_list.append(new_item)
self.set_val(self._ts_key(ts), ts_list)
self.current_id += 1
if update_hashes:
self.update_hashes(insert_idx)
self.set_current_id()
return insert_idx
def _ts_key(self, ts):
return 'ts%s' % ts
def get_by_timestamp(self, ts):
try:
return self.get_val(self._ts_key(ts))
except KeyError:
return []
def _bubble_up(self, start_idx):
if start_idx == None:
return
for idx in xrange(self.current_id, start_idx - 1, -1):
val = self.dict_store[idx]
self.dict_store[idx + 1] = val
def calculate_item_hash(self, item_data):
ts = item_data['ts']
uuid = item_data['uuid']
prev_item = self.get_previous_transaction(ts, uuid)
if prev_item is None:
prev_hash = ""
else:
prev_hash = prev_item['hash']
return self._get_item_hash(item_data, prev_hash)
def _get_item_hash(self, item_data, prev_hash):
# ts = item_data['ts']
# data = item_data['data']
# user = item_data['user']
# sig = item_data['signature']
m = md5()
m.update("%s%s%s" % (prev_hash, item_data['ts'], item_data['uuid']))
# m.update("%s%s%s%s%s" % (prev_hash, data, ts, user, sig))
return m.hexdigest()
def _get_prev_transaction_idx(self, ts, uuid):
idx = self.current_id
while idx > 0:
cmp_ts = self.dict_store[idx]['ts']
if cmp_ts > ts:
idx -= 1
elif cmp_ts == ts:
if self.dict_store[idx]['uuid'] > uuid:
idx -= 1
else:
break
else:
break
if idx == 0:
return None
return idx
def get_previous_transaction(self, ts, uuid):
idx = self._get_prev_transaction_idx(ts, uuid)
if idx is None:
return None
return self.dict_store[idx]
def get_top_hash(self):
try:
top_transaction = self.dict_store[self.current_id]
except KeyError:
return None
return top_transaction.get('hash', None)
def update_hashes(self, idx):
prev_item = self.dict_store[idx]
prev_hash = prev_item['hash']
idx += 1
while idx <= self.current_id:
item_data = self.dict_store[idx]
prev_hash = item_data['hash'] = self._get_item_hash(item_data, prev_hash)
self.dict_store[idx] = item_data
idx += 1
self.top_hash = prev_hash
def get_transaction(self, idx):
return self.dict_store[idx]
def resolve_missing_transactions(self, target_hash, transaction_gen):
lowest_idx = None
if target_hash != self.top_hash:
for batch_of_transactions in transaction_gen:
for transaction in batch_of_transactions:
transaction = copy.copy(transaction)
transaction.pop('hash', None)
list_of_transactions = self.get_by_timestamp(transaction['ts'])
if transaction not in list_of_transactions:
insert_idx = self.add_transaction(update_hashes=False, **transaction)
if lowest_idx is None or insert_idx < lowest_idx:
lowest_idx = insert_idx
if lowest_idx is not None:
self.update_hashes(lowest_idx)
if target_hash == self.top_hash:
break
def get_transactions_recent_to_old(self, batch_size=50):
idx = self.current_id
while idx > 0:
transactions = []
for i in xrange(idx, max(0, idx - batch_size), -1):
try:
transactions.append(self.get_transaction(i))
except KeyError:
continue
transactions.sort(key=lambda x: (x['ts'] * -1, x['uuid']))
idx -= batch_size
yield transactions
