class PostgreSQLProvider(): def __init__(self, logger=None): self.database = PooledPostgresqlExtDatabase( database=os.environ.get('DB_NAME'), user=os.environ.get('DB_USER'), password=os.environ.get('DB_PASSWORD'), host=os.environ.get('DB_HOST'), port=os.environ.get('DB_PORT'), max_connections=os.environ.get('DB_POOL_MAX_CONNECTIONS'), stale_timeout=os.environ.get('DB_POOL_TIMEOUT'), ) self.open_connection() self.logger = logger def __del__(self): self.close_connection() def open_connection(self): self.database.connect(reuse_if_open=True) print('Connected to PostgreSQL database') def close_connection(self): self.database.close() print('Connection to PostgreSQL database closed')
DB_PORT = environ.get('DUTY_DB_PORT', 5432) DB_PASS = environ.get('DUTY_DB_PASS', 'password') #pg_sql = PostgresqlDatabase(DB_NAME, user=DB_USER, password=DB_PASS, host=DB_HOST, port=DB_PORT, autocommit=True, autorollback=True) pg_sql = PooledPostgresqlExtDatabase( database=DB_NAME, user=DB_USER, password=DB_PASS, host=DB_HOST, port=DB_PORT, max_connections=8, stale_timeout=300, # 300 sec = 5 min autocommit=True, autorollback=True, ) pg_sql.close() class BaseModel(Model): class Meta: database = pg_sql # Пользователи class Users(BaseModel): name = CharField(unique=True) admin = BooleanField(default=False) timestamp = TimestampField(default=int(datetime.timestamp(datetime.now()))) sid = CharField(unique=True) token = CharField(null=True)
class Database(object): restored_database_integrity = False forging_delegates = [] def __init__(self): super().__init__() self.db = PooledPostgresqlExtDatabase( database=os.environ.get("POSTGRES_DB_NAME", "postgres"), user=os.environ.get("POSTGRES_DB_USER", "postgres"), host=os.environ.get("POSTGRES_DB_HOST", "127.0.0.1"), port=os.environ.get("POSTGRES_DB_PORT", "5432"), # password='******' autorollback=True, max_connections=32, stale_timeout=300, # 5 minutes ) # TODO: figure this out (try with creating a base class and only assigning # _meta.database to that base class) Block._meta.database = self.db Transaction._meta.database = self.db Round._meta.database = self.db PoolTransaction._meta.database = self.db self._active_delegates = [] self.wallets = WalletManager() def close(self): self.db.close() def get_last_block(self): """Get the last block Returns None if block can't be found. """ try: block = Block.select().order_by(Block.height.desc()).get() except Block.DoesNotExist: return None else: crypto_block = CryptoBlock.from_object(block) return crypto_block def save_block(self, block): logger.info("Saving block %s", block.id) if not isinstance(block, CryptoBlock): raise Exception("Block must be a type of crypto.objects.Block" ) # TODO: better exception with self.db.atomic() as db_txn: try: db_block = Block.from_crypto(block) db_block.save(force_insert=True) except Exception as e: # TODO: Make this not so broad! logger.error("Got an exception while saving a block") db_txn.rollback() logger.error(e) return with self.db.atomic() as db_txn: try: for transaction in block.transactions: db_transaction = Transaction.from_crypto(transaction) db_transaction.save(force_insert=True) except Exception as e: # TODO: Make this not so broad! logger.error("Got an exception while saving transactions") db_txn.rollback() db_block.delete_instance() logger.error(e) # TODO: replace with logger.error raise e def apply_round(self, height): next_height = 1 if height == 1 else height + 1 logger.info("Apply round next height: %s", next_height) current_round, _, max_delegates = calculate_round(next_height) logger.info("Current round %s", current_round) if next_height % max_delegates == 1: # TODO: Apparently forger can apply a round multiple times, so we need to # make sure that it only applies it once! Look at the code in ark core # to get the bigger picture of how it's done there logger.info("Starting round %s", current_round) # TODO: This is to update missed blocks on the wallet # self.update_delegate_stats(self.forging_delegates) # TODO: Save wallets to database # self.save_wallets # Get the active delegate list from in-memory wallet manager delegate_wallets = self.wallets.load_active_delegate_wallets( next_height) # TODO: ark core states that this is saving next round delegate list into # the db. Is that true? Or are we saving the current round delegate list # into the db? logger.info("STORING CURRENT ROUND %s", current_round) with self.db.atomic() as db_txn: try: for wallet in delegate_wallets: Round.create( public_key=wallet.public_key, balance=wallet.vote_balance, round=current_round, ) except Exception as e: # TODO: make this not so broad! logger.error("Got an exception while saving a round") db_txn.rollback() logger.error(e) raise e def apply_block(self, block): # TODO: implement this properly self.wallets.apply_block(block) # TODO: wat? # if (this.blocksInCurrentRound) { # this.blocksInCurrentRound.push(block); # } self.save_block(block) self.apply_round(block.height) # TODO: em wat? # // Check if we recovered from a fork # if (state.forkedBlock && # state.forkedBlock.data.height === this.block.data.height) { # this.logger.info("Successfully recovered from fork :star2:"); # state.forkedBlock = null; # } def verify_blockchain(self): """ Verify that the blockchain stored in the db is not corrupted This makes simple checks: - is last block available - is last block height equalt to the number of stored blocks - is the number of stored transactions equal to the number of sum of Block.number_of_transactions in the database - is the sum of all transaction fees equal to the sum of Block.total_fee - is the sum of all transaction amounts equal to the sum of Block.total_amount Returns a tuple (is_valid, errors) """ errors = [] last_block = self.get_last_block() block_stats = Block.statistics() transaction_stats = Transaction.statistics() if not last_block: errors.append("Last block is not available") else: if last_block.height != block_stats["blocks_count"]: errors.append( "Last block height: {}, number of stored blocks: {}". format(last_block.height, block_stats["blocks_count"])) # Number of stored transactions must be equal to the sum of # Block.number_of_transactions in the database if block_stats["transactions_count"] != transaction_stats[ "transactions_count"]: errors.append( ("Number of transactions: {}, " "number of transactions included in blocks: {}").format( transaction_stats["transactions_count"], block_stats["transactions_count"], )) # Sum of all transaction fees must equal to the sum of Block.total_fee if block_stats["total_fee"] != transaction_stats["total_fee"]: errors.append( ("Total transaction fees: {}, " "total transaction fees included in blocks: {}").format( transaction_stats["total_fee"], block_stats["total_fee"])) # Sum of all transaction amounts must equal to the sum of Block.total_amount if block_stats["total_amount"] != transaction_stats["total_amount"]: errors.append( ("Total transaction amounts: {}, " "total transaction amount included in blocks: {}").format( transaction_stats["total_amount"], block_stats["total_amount"])) is_valid = len(errors) == 0 return is_valid, errors def get_active_delegates(self, height): """Get the top 51 delegates TODO: this function is potentially very broken and returns all rounds? """ delegate_round, next_round, max_delegates = calculate_round(height) if not self._active_delegates or ( self._active_delegates and self._active_delegates[0].round != delegate_round): logger.info("Load delegates for round %s", delegate_round) delegates = list( Round.select().where(Round.round == delegate_round).order_by( Round.balance.desc(), Round.public_key.asc())) # for delegate in delegates: # wallet = self.wallets.find_by_public_key(delegate.public_key) # logger.info( # "%s %s %s %s", # delegate.public_key, # delegate.balance, # wallet.vote_balance, # wallet.username, # ) if delegates: seed = sha256(str(delegate_round).encode("utf-8")).digest() # TODO: Look into why we don't reorder every 5th element # (the second index += 1 # skips it). Also why do we create another seed, that is always the # same after the first run? # Apparently this order is used in forger. Might be better to put it # there instead of in a random function that doesn't tell you what it's # for, whatdoyouthink? index = 0 while index < len(delegates): for x in range(min(4, len(delegates) - index)): new_index = seed[x] % len(delegates) # Swap delegate on index with the delegate on new_index delegates[new_index], delegates[index] = ( delegates[index], delegates[new_index], ) index += 1 seed = sha256(seed).digest() index += 1 self._active_delegates = delegates return self._active_delegates def get_recent_block_ids(self): """Get 10 most recent block ids """ blocks = (Block.select(Block.id).order_by( Block.timestamp.desc()).limit(10).tuples()) return [x[0] for x in blocks] def get_block_by_id(self, block_id): try: block = Block.get(Block.id == block_id) except Block.DoesNotExist: return None else: return CryptoBlock.from_object(block) def get_forged_transaction_ids(self, transaction_ids): transactions = Transaction.select(Transaction.id).where( Transaction.id.in_(transaction_ids)) return [transaction.id for transaction in transactions] def transaction_is_forged(self, transaction_id): return Transaction.select().where( Transaction.id == transaction_id).exists() def get_blocks(self, height, limit, serialized, with_transactions=False): blocks = (Block.select().where( Block.height.between(height, height + limit)).order_by(Block.height.asc())) if with_transactions: block_ids = [block.id for block in blocks] transactions = (Transaction.select().where( Transaction.block_id.in_(block_ids)).order_by( Transaction.block_id.asc(), Transaction.sequence.asc())) transactions_map = defaultdict(list) for trans in transactions: # TODO: implement from_object on transaction and use that, instead of # creating it from serialized data. if serialized: transactions_map[trans.block_id].append(trans.serialized) else: transactions_map[trans.block_id].append( from_serialized(trans.serialized)) crypto_blocks = [] for block in blocks: crypto_block = CryptoBlock.from_object(block) if with_transactions: crypto_block.transactions = transactions_map[block.id] crypto_blocks.append(crypto_block) return crypto_blocks def get_blocks_by_id(self, block_ids): blocks = (Block.select().where(Block.id.in_(block_ids)).order_by( Block.height.desc())) return [CryptoBlock.from_object(block) for block in blocks] def get_blocks_by_heights(self, heights): if not isinstance(heights, list): raise Exception("heights must be a type of list") blocks = Block.select().where(Block.height.in_(heights)) return [CryptoBlock.from_object(block) for block in blocks] def delete_round(self, round_to_delete): Round.delete().where(Round.round == round_to_delete) def revert_block(self, block): current_round, next_round, max_delegates = calculate_round( block.height) if next_round == current_round + 1 and block.height > max_delegates: # const delegates = await this.calcPreviousActiveDelegates(round); # this.forgingDelegates = await this.getActiveDelegates(height, delegates); self.delete_round(next_round) self.wallets.revert_block(block) def rollback_to_round(self, to_round): # TODO: Get rid of this and use blockchain.revert_blocks instead """ Removes all rounds, block and transactions to the start of the `to_round`. NOTE: You need to restart the chain after calling this as this does not rollback wallets and any data that's in memory/redis cache. """ height = to_round * 51 block_select_query = Block.select( Block.id).where(Block.height >= height) transaction_query = Transaction.delete().where( Transaction.block_id.in_(block_select_query)) deleted_transactions = transaction_query.execute() logger.info("Deleted transactions: %s", deleted_transactions) block_query = Block.delete().where(Block.height >= height) deleted_blocks = block_query.execute() logger.info("Deleted blocks: %s", deleted_blocks) round_query = Round.delete().where(Round.round > to_round) deleted_rounds = round_query.execute() logger.info("Deleted rounds: %s", deleted_rounds)