def balance_legacy(db, address: str):
# from essentials.py
credit_ledger = Decimal(0)
q1 = db.execute("SELECT amount, reward FROM transactions WHERE recipient = ?", (address,))
entries = q1.fetchall()
for entry in entries:
credit_ledger += quantize_eight(entry[0]) + quantize_eight(entry[1])
debit_ledger = Decimal(0)
q2 = db.execute("SELECT amount, fee FROM transactions WHERE address = ?", (address,))
entries = q2.fetchall()
for entry in entries:
debit_ledger += quantize_eight(entry[0]) + quantize_eight(entry[1])
return quantize_eight(credit_ledger - debit_ledger)
def fee_calculate(openfield: str,
operation: str = '',
block: int = 0) -> Decimal:
# block var is no more needed, kept for interface retro compatibility
fee = Decimal("0.01") + (Decimal(len(openfield)) / Decimal("100000")
) # 0.01 dust
if operation == "token:issue":
fee = Decimal(fee) + Decimal("10")
if operation == "alias:register": # Take fee into account even if the protocol is not live yet.
fee = Decimal(fee) + Decimal("1")
elif openfield.startswith("alias="):
fee = Decimal(fee) + Decimal("1")
return quantize_eight(fee)
async def user_globalbalanceget(self, addresses):
"""Return total balance amounts from a list of addresses"""
# Sanitize, make sure addresses are addresses. Important here since we assemble the query by hand.
for address in addresses:
if not address_validate(address):
return "Error: bad address {}".format(address)
addresses = "('" + "','".join(addresses) + "')"
# print("add", addresses)
if self.config.direct_ledger:
if not self.ledger.legacy_db:
raise RuntimeError("V2 BD, Asking user_globalbalanceget")
base_mempool = await self.mempool.async_fetchall("SELECT amount, openfield, operation FROM transactions "
"WHERE address in {}".format(addresses))
# include mempool fees
debit_mempool = 0
if base_mempool:
for x in base_mempool:
debit_tx = Decimal(x[0])
fee = fee_calculate(x[1], x[2], 700001)
debit_mempool = quantize_eight(debit_mempool + debit_tx + fee)
else:
debit_mempool = 0
# include mempool fees
credit_ledger = Decimal("0")
for entry in await self.ledger.async_execute("SELECT amount FROM transactions WHERE recipient in{}"
.format(addresses)):
try:
credit_ledger = quantize_eight(credit_ledger) + quantize_eight(entry[0])
credit_ledger = 0 if credit_ledger is None else credit_ledger
except Exception:
credit_ledger = 0
fees = Decimal("0")
debit_ledger = Decimal("0")
for entry in await self.ledger.async_execute("SELECT fee, amount FROM transactions WHERE address in {}"
.format(addresses)):
try:
fees = quantize_eight(fees) + quantize_eight(entry[0])
fees = 0 if fees is None else fees
except Exception:
fees = 0
try:
debit_ledger = debit_ledger + Decimal(entry[1])
debit_ledger = 0 if debit_ledger is None else debit_ledger
except Exception:
debit_ledger = 0
debit = quantize_eight(debit_ledger + debit_mempool)
rewards = Decimal("0")
for entry in await self.ledger.async_execute("SELECT reward FROM transactions WHERE recipient in {}"
.format(addresses)):
try:
rewards = quantize_eight(rewards) + quantize_eight(entry[0])
rewards = 0 if rewards is None else rewards
except Exception:
rewards = 0
balance = quantize_eight(credit_ledger - debit - fees + rewards)
balance_no_mempool = (
float(credit_ledger)
- float(debit_ledger)
- float(fees)
+ float(rewards)
)
# app_log.info("Mempool: Projected transction address balance: " + str(balance))
return (
str(balance),
str(credit_ledger),
str(debit),
str(fees),
str(rewards),
str(balance_no_mempool),
)
else:
return "Error: Need direct ledger access or capable node"
async def user_balanceget(self, balance_address):
if self.config.direct_ledger and self.ledger.legacy_db:
if not self.ledger.legacy_db:
raise RuntimeError("V2 BD, Asking user_balanceget")
base_mempool = await self.mempool.async_fetchall("SELECT amount, openfield, operation FROM transactions "
"WHERE address = ?",
(balance_address, ))
# include mempool fees
debit_mempool = 0
if base_mempool:
for x in base_mempool:
debit_tx = Decimal(x[0])
fee = fee_calculate(x[1], x[2], 700001)
debit_mempool = quantize_eight(debit_mempool + debit_tx + fee)
else:
debit_mempool = 0
# include mempool fees
credit_ledger = Decimal("0")
for entry in await self.ledger.async_execute("SELECT amount FROM transactions WHERE recipient = ?",
(balance_address, )):
try:
credit_ledger = quantize_eight(credit_ledger) + quantize_eight(
entry[0]
)
credit_ledger = 0 if credit_ledger is None else credit_ledger
except Exception:
credit_ledger = 0
fees = Decimal("0")
debit_ledger = Decimal("0")
for entry in await self.ledger.async_execute("SELECT fee, amount FROM transactions WHERE address = ?",
(balance_address, )):
try:
fees = quantize_eight(fees) + quantize_eight(entry[0])
fees = 0 if fees is None else fees
except Exception:
fees = 0
try:
debit_ledger = debit_ledger + Decimal(entry[1])
debit_ledger = 0 if debit_ledger is None else debit_ledger
except Exception:
debit_ledger = 0
debit = quantize_eight(debit_ledger + debit_mempool)
rewards = Decimal("0")
for entry in await self.ledger.async_execute("SELECT reward FROM transactions WHERE recipient = ?",
(balance_address, )):
try:
rewards = quantize_eight(rewards) + quantize_eight(entry[0])
rewards = 0 if rewards is None else rewards
except Exception:
rewards = 0
balance = quantize_eight(credit_ledger - debit - fees + rewards)
balance_no_mempool = (
float(credit_ledger)
- float(debit_ledger)
- float(fees)
+ float(rewards)
)
# app_log.info("Mempool: Projected transction address balance: " + str(balance))
return (
str(balance),
str(credit_ledger),
str(debit),
str(fees),
str(rewards),
str(balance_no_mempool),
)
else:
stream = await self._node_stream()
try:
await self._send("balanceget", stream)
await self._send(balance_address, stream)
balance = await self._receive(stream)
return balance
except KeyboardInterrupt:
stream.close()
finally:
if stream:
stream.close()
def merge(self,
data,
peer_ip,
c,
size_bypass=False,
wait=False,
revert=False):
"""
Checks and merge the tx list in out mempool
:param data:
:param peer_ip:
:param c:
:param size_bypass: if True, will merge whatever the mempool size is
:param wait: if True, will wait until the main db_lock is free. if False, will just drop.
:param revert: if True, we are reverting tx from digest_block, so main lock is on. Don't bother, process without lock.
:return:
"""
global REFUSE_OLDER_THAN
# Easy cases of empty or invalid data
if not data:
return "Mempool from {} was empty".format(peer_ip)
mempool_result = []
if data == '*':
raise ValueError("Connection lost")
try:
if self.peers_sent[peer_ip] > time.time(
) and peer_ip != '127.0.0.1':
self.app_log.warning(
"Mempool ignoring merge from frozen {}".format(peer_ip))
mempool_result.append(
"Mempool ignoring merge from frozen {}".format(peer_ip))
return mempool_result
except:
# unknown peer
pass
if not essentials.is_sequence(data):
if peer_ip != '127.0.0.1':
with self.peers_lock:
self.peers_sent[peer_ip] = time.time() + 10 * 60
self.app_log.warning(
"Freezing mempool from {} for 10 min - Bad TX format".
format(peer_ip))
mempool_result.append("Bad TX Format")
return mempool_result
if not revert:
while self.db_lock.locked():
# prevent transactions which are just being digested from being added to mempool
if not wait:
# not reverting, but not waiting, bye
# By default, we don't wait.
mempool_result.append("Locked ledger, dropping txs")
return mempool_result
self.app_log.warning(
"Waiting for block digestion to finish before merging mempool"
)
time.sleep(1)
# if reverting, don't bother with main lock, go on.
# Let's really dig
mempool_result.append(
"Mempool merging started from {}".format(peer_ip))
# Single time reference here for the whole merge.
time_now = time.time()
# calculate current mempool size before adding txs
mempool_size = self.size()
# TODO: we check main ledger db is not locked before beginning, but we don't lock? ok, see comment in node.py. since it's called from a lock, it would deadlock.
# merge mempool
# while self.lock.locked():
# time.sleep(1)
with self.lock:
try:
block_list = data
if not isinstance(
block_list[0], list
): # convert to list of lists if only one tx and not handled
block_list = [block_list]
for transaction in block_list:
if size_bypass or self.space_left_for_tx(
transaction, mempool_size):
# all transactions in the mempool need to be cycled to check for special cases,
# therefore no while/break loop here
mempool_timestamp = '%.2f' % (quantize_two(
transaction[0]))
mempool_timestamp_float = float(
transaction[0]) # limit Decimal where not needed
mempool_address = str(transaction[1])[:56]
mempool_recipient = str(transaction[2])[:56]
mempool_amount = '%.8f' % (quantize_eight(
transaction[3])) # convert scientific notation
mempool_amount_float = float(transaction[3])
mempool_signature_enc = str(transaction[4])[:684]
mempool_public_key_hashed = str(transaction[5])[:1068]
if "b'" == mempool_public_key_hashed[:2]:
mempool_public_key_hashed = transaction[5][2:1070]
mempool_operation = str(transaction[6])[:30]
mempool_openfield = str(transaction[7])[:100000]
# Begin with the easy tests that do not require cpu or disk access
if mempool_amount_float < 0:
mempool_result.append(
"Mempool: Negative balance spend attempt")
continue
if not essentials.address_validate(mempool_address):
mempool_result.append(
"Mempool: Invalid address {}".format(
mempool_address))
continue
if not essentials.address_validate(mempool_recipient):
mempool_result.append(
"Mempool: Invalid recipient {}".format(
mempool_recipient))
continue
if mempool_timestamp_float > time_now:
mempool_result.append(
"Mempool: Future transaction rejected {}s".
format(mempool_timestamp_float - time_now))
continue
if mempool_timestamp_float < time_now - REFUSE_OLDER_THAN:
# don't accept old txs, mempool needs to be harsher than ledger
mempool_result.append(
"Mempool: Too old a transaction")
continue
# Then more cpu heavy tests
hashed_address = hashlib.sha224(
base64.b64decode(
mempool_public_key_hashed)).hexdigest()
if mempool_address != hashed_address:
mempool_result.append(
"Mempool: Attempt to spend from a wrong address {} instead of {}"
.format(mempool_address, hashed_address))
continue
# Crypto tests - more cpu hungry
try:
essentials.validate_pem(mempool_public_key_hashed)
except ValueError as e:
mempool_result.append(
"Mempool: Public key does not validate: {}".
format(e))
# recheck sig
try:
mempool_public_key = RSA.importKey(
base64.b64decode(mempool_public_key_hashed))
mempool_signature_dec = base64.b64decode(
mempool_signature_enc)
verifier = PKCS1_v1_5.new(mempool_public_key)
tx_signed = (mempool_timestamp, mempool_address,
mempool_recipient, mempool_amount,
mempool_operation, mempool_openfield)
my_hash = SHA.new(str(tx_signed).encode("utf-8"))
if not verifier.verify(my_hash,
mempool_signature_dec):
mempool_result.append(
"Mempool: Wrong signature ({}) for data {} in mempool insert attempt"
.format(mempool_signature_enc, tx_signed))
continue
except Exception as e:
mempool_result.append(
"Mempool: Unexpected error checking sig: {}".
format(e))
continue
# Only now, process the tests requiring db access
mempool_in = self.sig_check(mempool_signature_enc)
# Temp: get last block for HF reason
essentials.execute_param_c(
c,
"SELECT block_height FROM transactions WHERE 1 ORDER by block_height DESC limit ?",
(1, ), self.app_log)
last_block = c.fetchone()[0]
# reject transactions which are already in the ledger
# TODO: not clean, will need to have ledger as a module too.
# TODO: need better txid index, this is very sloooooooow
essentials.execute_param_c(
c,
"SELECT timestamp FROM transactions WHERE signature = ?",
(mempool_signature_enc, ), self.app_log)
ledger_in = bool(c.fetchone())
# remove from mempool if it's in both ledger and mempool already
if mempool_in and ledger_in:
try:
# Do not lock, we already have the lock for the whole merge.
self.execute(SQL_DELETE_TX,
(mempool_signature_enc, ))
self.commit()
mempool_result.append(
"Mempool: Transaction deleted from our mempool"
)
except: # experimental try and except
mempool_result.append(
"Mempool: Transaction was not present in the pool anymore"
)
continue
if ledger_in:
mempool_result.append(
"That transaction is already in our ledger")
# Can be a syncing node. Do not request mempool from this peer until FREEZE_MIN min
# ledger_in is the ts of the tx in ledger. if it's recent, maybe the peer is just one block late.
# give him 3 minute margin.
if (peer_ip != '127.0.0.1') and (
ledger_in < time_now - 60 * 3):
with self.peers_lock:
self.peers_sent[peer_ip] = time.time(
) + FREEZE_MIN * 60
self.app_log.warning(
"Freezing mempool from {} for {} min.".
format(peer_ip, FREEZE_MIN))
# Here, we point blank stop processing the batch from this host since it's outdated.
# Update: Do not, since it blocks further valid tx - case has been found in real use.
# return mempool_result
continue
# Already there, just ignore then
if mempool_in:
mempool_result.append(
"That transaction is already in our mempool")
continue
# Here we covered the basics, the current tx is conform and signed. Now let's check balance.
# verify balance
mempool_result.append(
"Mempool: Received address: {}".format(
mempool_address))
# include mempool fees
result = self.fetchall(
"SELECT amount, openfield, operation FROM transactions WHERE address = ?",
(mempool_address, ))
debit_mempool = 0
if result:
for x in result:
debit_tx = quantize_eight(x[0])
fee = fee_calculate(x[1], x[2], last_block)
debit_mempool = quantize_eight(debit_mempool +
debit_tx + fee)
credit = 0
for entry in essentials.execute_param_c(
c,
"SELECT amount FROM transactions WHERE recipient = ?",
(mempool_address, ), self.app_log):
credit = quantize_eight(credit) + quantize_eight(
entry[0])
debit_ledger = 0
for entry in essentials.execute_param_c(
c,
"SELECT amount FROM transactions WHERE address = ?",
(mempool_address, ), self.app_log):
debit_ledger = quantize_eight(
debit_ledger) + quantize_eight(entry[0])
debit = debit_ledger + debit_mempool
fees = 0
for entry in essentials.execute_param_c(
c,
"SELECT fee FROM transactions WHERE address = ?",
(mempool_address, ), self.app_log):
fees = quantize_eight(fees) + quantize_eight(
entry[0])
rewards = 0
for entry in essentials.execute_param_c(
c,
"SELECT sum(reward) FROM transactions WHERE recipient = ?",
(mempool_address, ), self.app_log):
rewards = quantize_eight(rewards) + quantize_eight(
entry[0])
balance = quantize_eight(
credit - debit - fees + rewards -
quantize_eight(mempool_amount))
balance_pre = quantize_eight(credit - debit_ledger -
fees + rewards)
fee = fee_calculate(mempool_openfield,
mempool_operation, last_block)
if quantize_eight(mempool_amount) > quantize_eight(
balance_pre
): #mp amount is already included in "balance" var! also, that tx might already be in the mempool
mempool_result.append(
"Mempool: Sending more than owned")
continue
if quantize_eight(balance) - quantize_eight(fee) < 0:
mempool_result.append(
"Mempool: Cannot afford to pay fees")
continue
# Pfew! we can finally insert into mempool - all is str, type converted and enforced above
self.execute(
"INSERT INTO transactions VALUES (?,?,?,?,?,?,?,?,?)",
(mempool_timestamp, mempool_address,
mempool_recipient, mempool_amount,
mempool_signature_enc, mempool_public_key_hashed,
mempool_operation, mempool_openfield,
int(time_now)))
mempool_result.append(
"Mempool updated with a received transaction from {}"
.format(peer_ip))
mempool_result.append("Success")
self.commit(
) # Save (commit) the changes to mempool db
mempool_size += sys.getsizeof(
str(transaction)) / 1000000.0
else:
mempool_result.append(
"Local mempool is already full for this tx type, skipping merging"
)
# self.app_log.warning("Local mempool is already full for this tx type, skipping merging")
# TEMP
# print("Mempool insert", mempool_result)
return mempool_result
# TODO: Here maybe commit() on c to release the write lock?
except Exception as e:
self.app_log.warning("Mempool: Error processing: {} {}".format(
data, e))
if self.config.debug_conf == 1:
raise
return mempool_result