class ManagerBase(object): def __init__(self, filename): self.storage = DictStore(filename) if 'vals' not in self.storage: self.storage['vals'] = dict() def _clear(self): self.storage.clear() self.storage['vals'] = dict() @property def vals(self): """ Returns a copy of the resources dict """ return self.storage['vals'] @update_storage def add_item(self, vals, item, name=None): if name is None: name = item.name vals[name] = item @update_storage def remove_item(self, vals, name): del vals[name]
class TransactionManager(object): def __init__(self, filename='transaction_storage.pickle'): self.dict_store = DictStore(filename) self.current_id = self.get_current_id() self.top_hash = self.get_top_hash() def _clear(self): self.current_id = 0 self.dict_store.clear() self.top_hash = None def get_current_id(self): try: return self.get_val("cur_id") except KeyError: return 0 def set_val(self, key, val): self.dict_store[key] = {"val": val} def get_val(self, key): return self.dict_store[key]['val'] def set_current_id(self): self.set_val("cur_id", self.current_id) def validate_user_signature(self, user, signature): """ Validates the user's signature. Raises an exception on failure :param user: user :param signature: user signature """ def add_transaction(self, data, ts, user, signature, uuid=None, update_hashes=True): if uuid is None: uuid = uuid4() insert_idx = self._get_prev_transaction_idx(ts, uuid) if insert_idx is None: insert_idx = 1 else: insert_idx += 1 self._bubble_up(insert_idx) self.validate_user_signature(user, signature) new_item = { "data": data, "ts": ts, "user": user, "signature": signature, "uuid": uuid, } hash = self.calculate_item_hash(new_item) new_item['hash'] = hash self.dict_store[insert_idx] = new_item ts_list = self.get_by_timestamp(ts) new_item.pop('hash', None) ts_list.append(new_item) self.set_val(self._ts_key(ts), ts_list) self.current_id += 1 if update_hashes: self.update_hashes(insert_idx) self.set_current_id() return insert_idx def _ts_key(self, ts): return 'ts%s' % ts def get_by_timestamp(self, ts): try: return self.get_val(self._ts_key(ts)) except KeyError: return [] def _bubble_up(self, start_idx): if start_idx == None: return for idx in xrange(self.current_id, start_idx - 1, -1): val = self.dict_store[idx] self.dict_store[idx + 1] = val def calculate_item_hash(self, item_data): ts = item_data['ts'] uuid = item_data['uuid'] prev_item = self.get_previous_transaction(ts, uuid) if prev_item is None: prev_hash = "" else: prev_hash = prev_item['hash'] return self._get_item_hash(item_data, prev_hash) def _get_item_hash(self, item_data, prev_hash): # ts = item_data['ts'] # data = item_data['data'] # user = item_data['user'] # sig = item_data['signature'] m = md5() m.update("%s%s%s" % (prev_hash, item_data['ts'], item_data['uuid'])) # m.update("%s%s%s%s%s" % (prev_hash, data, ts, user, sig)) return m.hexdigest() def _get_prev_transaction_idx(self, ts, uuid): idx = self.current_id while idx > 0: cmp_ts = self.dict_store[idx]['ts'] if cmp_ts > ts: idx -= 1 elif cmp_ts == ts: if self.dict_store[idx]['uuid'] > uuid: idx -= 1 else: break else: break if idx == 0: return None return idx def get_previous_transaction(self, ts, uuid): idx = self._get_prev_transaction_idx(ts, uuid) if idx is None: return None return self.dict_store[idx] def get_top_hash(self): try: top_transaction = self.dict_store[self.current_id] except KeyError: return None return top_transaction.get('hash', None) def update_hashes(self, idx): prev_item = self.dict_store[idx] prev_hash = prev_item['hash'] idx += 1 while idx <= self.current_id: item_data = self.dict_store[idx] prev_hash = item_data['hash'] = self._get_item_hash(item_data, prev_hash) self.dict_store[idx] = item_data idx += 1 self.top_hash = prev_hash def get_transaction(self, idx): return self.dict_store[idx] def resolve_missing_transactions(self, target_hash, transaction_gen): lowest_idx = None if target_hash != self.top_hash: for batch_of_transactions in transaction_gen: for transaction in batch_of_transactions: transaction = copy.copy(transaction) transaction.pop('hash', None) list_of_transactions = self.get_by_timestamp(transaction['ts']) if transaction not in list_of_transactions: insert_idx = self.add_transaction(update_hashes=False, **transaction) if lowest_idx is None or insert_idx < lowest_idx: lowest_idx = insert_idx if lowest_idx is not None: self.update_hashes(lowest_idx) if target_hash == self.top_hash: break def get_transactions_recent_to_old(self, batch_size=50): idx = self.current_id while idx > 0: transactions = [] for i in xrange(idx, max(0, idx - batch_size), -1): try: transactions.append(self.get_transaction(i)) except KeyError: continue transactions.sort(key=lambda x: (x['ts'] * -1, x['uuid'])) idx -= batch_size yield transactions