def balance_from_cursor(cursor, address):
credit = Decimal("0")
debit = Decimal("0")
for entry in cursor.execute(
"SELECT amount,reward FROM transactions WHERE recipient = ? ",
(address, )):
try:
#result = cursor.fetchall()
credit = credit + quantize_eight(entry[0]) + quantize_eight(
entry[1])
#print (result)
credit = 0 if credit is None else credit
except Exception as e:
credit = 0
#print (credit)
for entry in cursor.execute(
"SELECT amount,fee FROM transactions WHERE address = ? ",
(address, )):
try:
# result = cursor.fetchall()
debit = debit + quantize_eight(entry[0]) + quantize_eight(entry[1])
# print (result)
debit = 0 if debit is None else debit
except Exception as e:
debit = 0
# print (debit)
return quantize_eight(credit - debit)
def ledger_balance3(address, cache, db_handler):
# Many heavy blocks are pool payouts, same address.
# Cache pre_balance instead of recalc for every tx
if address in cache:
return cache[address]
credit_ledger = Decimal(0)
db_handler.execute_param(
db_handler.c,
"SELECT amount, reward FROM transactions WHERE recipient = ?;",
(address, ))
entries = db_handler.c.fetchall()
for entry in entries:
credit_ledger += quantize_eight(entry[0]) + quantize_eight(entry[1])
debit_ledger = Decimal(0)
db_handler.execute_param(
db_handler.c,
"SELECT amount, fee FROM transactions WHERE address = ?;", (address, ))
entries = db_handler.c.fetchall()
for entry in entries:
debit_ledger += quantize_eight(entry[0]) + quantize_eight(entry[1])
cache[address] = quantize_eight(credit_ledger - debit_ledger)
return cache[address]
def ledger_balance2(address, c):
# 2 sql requests only instead of 4 + more rational quantize use.
credit_ledger = Decimal(0)
for entry in execute_param(c, "SELECT amount, reward FROM transactions WHERE recipient = ?;", (address,)):
credit_ledger += quantize_eight(entry[0]) + quantize_eight(entry[1])
debit_ledger = Decimal(0)
for entry in execute_param(c, "SELECT amount, fee FROM transactions WHERE address = ?;", (address,)):
debit_ledger += quantize_eight(entry[0]) + quantize_eight(entry[1])
return quantize_eight(credit_ledger - debit_ledger)
def staking_revalidate(conn, c, index, index_cursor, block, app_log):
"remove nodes that removed balance, to be run every 10k blocks"
index_cursor.execute("SELECT * FROM staking")
staking = index_cursor.fetchall()
for staking in staking:
app_log.warning(staking)
address = staking[2]
app_log.warning("address: {}".format(address))
balance_savings = staking[3]
app_log.warning("balance_savings: {}".format(balance_savings))
balance = balanceget_at_block(address, block, c)
app_log.warning("balance: {}".format(balance))
if quantize_eight(balance) < 10000:
index_cursor.execute("DELETE FROM staking WHERE address = ?",
(address, ))
index.commit()
else: #update balance
index_cursor.execute(
"UPDATE staking SET balance = ? WHERE address = ?",
(balance, address))
index.commit()
app_log.warning(
"staking balance updated from {} to {} for {}".format(
balance_savings, balance, address))
def staking_payout(conn, c, index, index_cursor, block_height, timestamp,
app_log):
"payout, to be run every 10k blocks"
index_cursor.execute("SELECT * FROM staking")
staking = index_cursor.fetchall()
app_log.warning("staking: {}".format(staking))
staking_total = len(staking)
mirror_hash = mirror_hash_generate(c)
for staking in staking:
block_staking = staking[0]
if block_staking <= block_height:
address = staking[2]
balance_savings = staking[3]
app_log.warning("balance_savings: {}".format(balance_savings))
stake = str(
quantize_eight(percentage(100 / staking_total,
balance_savings)))
app_log.warning("stake: {}".format(stake))
try:
c.execute(
"SELECT * from transactions WHERE block_height = ? AND recipient = ?",
(
-block_height,
address,
))
dummy = c.fetchall()[0] # check for uniqueness
app_log.warning(
"staking payout already processed: {} {}".format(
block_height, address))
except:
"""skip direct bis payouts
c.execute("INSERT INTO transactions VALUES (?,?,?,?,?,?,?,?,?,?,?,?)",(-block_height,timestamp,"staking",address,stake,"0","0",mirror_hash,"0","0","mnpayout","0"))
conn.commit()
app_log.warning ("staking payout added: {} {}".format (block_height, address))
"""
#fuel
stake_int = int(float(stake))
if stake_int < 1:
stake_int = 1
c.execute(
"INSERT INTO transactions VALUES (?,?,?,?,?,?,?,?,?,?,?,?)",
(-block_height, timestamp, "staking", address, "0", "0",
"0", mirror_hash, "0", "0", "token:transfer",
"fuel:{}".format(stake_int)))
conn.commit()
app_log.warning("Staking fuel payout added: {} {}".format(
block_height, address))
#fuel
else:
app_log.warning("staking is registered ahead of current block")
def staking_update(conn, c, index, index_cursor, mode, reg_phase_end, app_log):
"""update register of staking based on the current phase (10000 block intervals)"""
if mode not in ("normal", "reindex"):
raise ValueError("Wrong value for staking_update function")
check_db(index, index_cursor)
if mode == "reindex":
app_log.warning("staking database will be reindexed")
index_cursor.execute("DELETE FROM staking")
index.commit()
reg_phase_start = reg_phase_end - 10000
app_log.warning("reg_phase_start: {}".format(reg_phase_start))
app_log.warning("reg_phase_end: {}".format(reg_phase_end))
c.execute(
"SELECT block_height, timestamp, address, recipient,operation, openfield FROM transactions WHERE block_height >= ? AND block_height <= ? AND operation = ? ORDER BY block_height, timestamp LIMIT 100",
(
reg_phase_start,
reg_phase_end,
"staking:register",
))
results = c.fetchall() #more efficient than "for row in"
for row in results:
try:
block_height = row[0]
timestamp = row[1]
address = row[2]
try:
index_cursor.execute("SELECT * from staking WHERE address = ?",
(address, ))
dummy = index_cursor.fetchall()[0] #check for uniqueness
app_log.warning(
"staking already registered: {}".format(address))
except:
app_log.warning("address: {}".format(address))
balance = balanceget_at_block(address, reg_phase_end, c)
if quantize_eight(balance) >= 10000:
index_cursor.execute(
"INSERT INTO staking VALUES (?, ?, ?, ?)",
(block_height, timestamp, address, balance))
index.commit()
app_log.warning("Staking added: {} {}".format(
block_height, address))
else:
app_log.warning("Insufficient balance for staking")
except Exception as e:
app_log.warning(
"Staking registration ran into the following issue: {}".format(
e))
return reg_phase_start, reg_phase_end
def fee_calculate(openfield: str,
operation: str = '',
block: int = 0) -> Decimal:
# block var will be removed after HF
fee = Decimal("0.01") + (Decimal(len(openfield)) / Decimal("100000")
) # 0.01 dust
if operation == "token:issue":
fee = Decimal(fee) + Decimal("10")
if openfield.startswith("alias="):
fee = Decimal(fee) + Decimal("1")
#if operation == "alias:register": #add in the future, careful about forking
# fee = Decimal(fee) + Decimal("1")
return quantize_eight(fee)
def ledger_balance_node(address, c):
credit_ledger = Decimal("0")
for entry in execute_param(c, "SELECT amount FROM transactions WHERE recipient = ?;", (address,)):
credit_ledger = quantize_eight(credit_ledger) + quantize_eight(entry[0])
credit_ledger = 0 if credit_ledger is None else quantize_eight(credit_ledger)
debit_ledger = Decimal("0")
for entry in execute_param(c, "SELECT amount FROM transactions WHERE address = ?;", (address,)):
debit_ledger = quantize_eight(debit_ledger) + quantize_eight(entry[0])
debit_ledger = 0 if debit_ledger is None else quantize_eight(debit_ledger)
fees = Decimal("0")
for entry in execute_param(c, "SELECT fee FROM transactions WHERE address = ?;", (address,)):
try:
fees = quantize_eight(fees) + quantize_eight(entry[0])
fees = 0 if fees is None else fees
except:
fees = 0
rewards = Decimal("0")
for entry in execute_param(c, "SELECT reward FROM transactions WHERE recipient = ?;", (address,)):
try:
rewards = quantize_eight(rewards) + quantize_eight(entry[0])
rewards = 0 if rewards is None else rewards
except:
rewards = 0
return quantize_eight(credit_ledger - debit_ledger + rewards - fees)
def balanceget_at_block(balance_address, block, h3):
# verify balance
credit_ledger = Decimal("0")
for entry in execute_param(
h3,
("SELECT amount FROM transactions WHERE block_height <= ? AND block_height >= ? AND recipient = ?;"
), (
block,
-block,
balance_address,
)):
try:
credit_ledger = quantize_eight(credit_ledger) + quantize_eight(
entry[0])
credit_ledger = 0 if credit_ledger is None else credit_ledger
except:
credit_ledger = 0
fees = Decimal("0")
debit_ledger = Decimal("0")
for entry in execute_param(
h3,
("SELECT fee, amount FROM transactions WHERE block_height <= ? AND block_height >= ? AND address = ?;"
), (
block,
-block,
balance_address,
)):
try:
fees = quantize_eight(fees) + quantize_eight(entry[0])
fees = 0 if fees is None else fees
except:
fees = 0
try:
debit_ledger = debit_ledger + Decimal(entry[1])
debit_ledger = 0 if debit_ledger is None else debit_ledger
except:
debit_ledger = 0
debit = quantize_eight(debit_ledger)
rewards = Decimal("0")
for entry in execute_param(
h3,
("SELECT reward FROM transactions WHERE block_height <= ? AND block_height >= ? AND recipient = ?;"
), (
block,
-block,
balance_address,
)):
try:
rewards = quantize_eight(rewards) + quantize_eight(entry[0])
rewards = 0 if rewards is None else rewards
except:
rewards = 0
balance = quantize_eight(credit_ledger - debit - fees + rewards)
# app_log.info("Mempool: Projected transction address balance: " + str(balance))
return str(
balance
) #, str (credit_ledger), str (debit), str (fees), str (rewards)
def merge(self,
data: list,
peer_ip: str,
c,
size_bypass: bool = False,
wait: bool = False,
revert: bool = False) -> list:
"""
Checks and merge the tx list in out mempool
:param data:
:param peer_ip:
:param c:
:param size_bypass: if True, will merge whatever the mempool size is
:param wait: if True, will wait until the main db_lock is free. if False, will just drop.
:param revert: if True, we are reverting tx from digest_block, so main lock is on. Don't bother, process without lock.
:return:
"""
global REFUSE_OLDER_THAN
# Easy cases of empty or invalid data
if not data:
return ["Mempool from {} was empty".format(peer_ip)]
mempool_result = []
if data == '*':
raise ValueError("Connection lost")
try:
if self.peers_sent[peer_ip] > time.time(
) and peer_ip != '127.0.0.1':
self.app_log.warning(
"Mempool ignoring merge from frozen {}".format(peer_ip))
mempool_result.append(
"Mempool ignoring merge from frozen {}".format(peer_ip))
return mempool_result
except:
# unknown peer
pass
if not essentials.is_sequence(data):
if peer_ip != '127.0.0.1':
with self.peers_lock:
self.peers_sent[peer_ip] = time.time() + 10 * 60
self.app_log.warning(
"Freezing mempool from {} for 10 min - Bad TX format".
format(peer_ip))
mempool_result.append("Bad TX Format")
return mempool_result
if not revert:
while self.db_lock.locked():
# prevent transactions which are just being digested from being added to mempool
if not wait:
# not reverting, but not waiting, bye
# By default, we don't wait.
mempool_result.append("Locked ledger, dropping txs")
return mempool_result
self.app_log.warning(
"Waiting for block digestion to finish before merging mempool"
)
time.sleep(1)
# if reverting, don't bother with main lock, go on.
# Let's really dig
mempool_result.append(
"Mempool merging started from {}".format(peer_ip))
# Single time reference here for the whole merge.
time_now = time.time()
# calculate current mempool size before adding txs
mempool_size = self.size()
# TODO: we check main ledger db is not locked before beginning, but we don't lock? ok, see comment in node.py. since it's called from a lock, it would deadlock.
# merge mempool
# while self.lock.locked():
# time.sleep(1)
with self.lock:
try:
block_list = data
if not isinstance(
block_list[0], list
): # convert to list of lists if only one tx and not handled
block_list = [block_list]
for transaction in block_list:
if size_bypass or self.space_left_for_tx(
transaction, mempool_size):
# all transactions in the mempool need to be cycled to check for special cases,
# therefore no while/break loop here
try:
mempool_timestamp = '%.2f' % (quantize_two(
transaction[0]))
mempool_timestamp_float = float(
transaction[0]
) # limit Decimal where not needed
except Exception as e:
mempool_result.append(
"Mempool: Invalid timestamp {}".format(
transaction[0]))
if not essentials.address_validate(transaction[1]):
mempool_result.append(
"Mempool: Invalid address {}".format(
transaction[1]))
continue
# We could now ignore the truncates here, I left them for explicit reminder of the various fields max lengths.
mempool_address = str(transaction[1])[:56]
if not essentials.address_validate(transaction[2]):
mempool_result.append(
"Mempool: Invalid recipient {}".format(
transaction[2]))
continue
mempool_recipient = str(transaction[2])[:56]
try:
mempool_amount = '%.8f' % (quantize_eight(
transaction[3])) # convert scientific notation
mempool_amount_float = float(transaction[3])
except Exception as e:
mempool_result.append(
"Mempool: Invalid amount {}".format(
transaction[3]))
continue
if len(transaction[4]) > 684:
mempool_result.append(
"Mempool: Invalid signature len{}".format(
len(transaction[4])))
continue
mempool_signature_enc = str(transaction[4])[:684]
if len(transaction[5]) > 1068:
mempool_result.append(
"Mempool: Invalid pubkey len{}".format(
len(transaction[5])))
continue
mempool_public_key_b64encoded = str(
transaction[5])[:1068]
if "b'" == mempool_public_key_b64encoded[:2]:
# Binary content instead of str - leftover from legacy code?
mempool_public_key_b64encoded = transaction[5][
2:1070]
if len(transaction[6]) > 30:
mempool_result.append(
"Mempool: Invalid operation len{}".format(
len(transaction[6])))
continue
mempool_operation = str(transaction[6])[:30]
if len(transaction[7]) > 100000:
mempool_result.append(
"Mempool: Invalid openfield len{}".format(
len(transaction[7])))
continue
mempool_openfield = str(transaction[7])[:100000]
if len(mempool_openfield) <= 4:
# no or short message for a mandatory message
if mempool_recipient in self.config.mandatory_message.keys(
):
mempool_result.append(
"Mempool: Missing message - {}".format(
self.config.
mandatory_message[mempool_recipient]))
continue
# Begin with the easy tests that do not require cpu or disk access
if mempool_amount_float < 0:
mempool_result.append(
"Mempool: Negative balance spend attempt")
continue
if mempool_timestamp_float > time_now:
mempool_result.append(
"Mempool: Future transaction rejected {}s".
format(mempool_timestamp_float - time_now))
continue
if mempool_timestamp_float < time_now - REFUSE_OLDER_THAN:
# don't accept old txs, mempool needs to be harsher than ledger
mempool_result.append(
"Mempool: Too old a transaction")
continue
# Then more cpu heavy tests
buffer = str((mempool_timestamp, mempool_address,
mempool_recipient, mempool_amount,
mempool_operation,
mempool_openfield)).encode("utf-8")
# Will raise if error
try:
SignerFactory.verify_bis_signature(
mempool_signature_enc,
mempool_public_key_b64encoded, buffer,
mempool_address)
except Exception as e:
mempool_result.append(
f"Mempool: Signature did not match for address ({e})"
)
continue
# Only now, process the tests requiring db access
mempool_in = self.sig_check(mempool_signature_enc)
# Temp: get last block for HF reason
essentials.execute_param_c(
c,
"SELECT block_height FROM transactions WHERE 1 ORDER by block_height DESC limit ?",
(1, ), self.app_log)
last_block = c.fetchone()[0]
# reject transactions which are already in the ledger
# TODO: not clean, will need to have ledger as a module too.
# TODO: need better txid index, this is very sloooooooow
if self.config.old_sqlite:
essentials.execute_param_c(
c,
"SELECT timestamp FROM transactions WHERE signature = ?1",
(mempool_signature_enc, ), self.app_log)
else:
essentials.execute_param_c(
c,
"SELECT timestamp FROM transactions WHERE substr(signature,1,4) = substr(?1,1,4) AND signature = ?1",
(mempool_signature_enc, ), self.app_log)
ledger_in = bool(c.fetchone())
# remove from mempool if it's in both ledger and mempool already
if mempool_in and ledger_in:
try:
# Do not lock, we already have the lock for the whole merge.
if self.config.old_sqlite:
self.execute(SQL_DELETE_TX_OLD,
(mempool_signature_enc, ))
else:
self.execute(SQL_DELETE_TX,
(mempool_signature_enc, ))
self.commit()
mempool_result.append(
"Mempool: Transaction deleted from our mempool"
)
except: # experimental try and except
mempool_result.append(
"Mempool: Transaction was not present in the pool anymore"
)
continue
if ledger_in:
mempool_result.append(
"That transaction is already in our ledger")
# Can be a syncing node. Do not request mempool from this peer until FREEZE_MIN min
# ledger_in is the ts of the tx in ledger. if it's recent, maybe the peer is just one block late.
# give him 15 minute margin.
if (peer_ip != '127.0.0.1') and (
ledger_in < time_now - 60 * 15):
with self.peers_lock:
self.peers_sent[peer_ip] = time.time(
) + FREEZE_MIN * 60
self.app_log.warning(
"Freezing mempool from {} for {} min.".
format(peer_ip, FREEZE_MIN))
# Here, we point blank stop processing the batch from this host since it's outdated.
# Update: Do not, since it blocks further valid tx - case has been found in real use.
# return mempool_result
continue
# Already there, just ignore then
if mempool_in:
mempool_result.append(
"That transaction is already in our mempool")
continue
# Here we covered the basics, the current tx is conform and signed. Now let's check balance.
# verify balance
mempool_result.append(
"Mempool: Received address: {}".format(
mempool_address))
# include mempool fees
result = self.fetchall(
"SELECT amount, openfield, operation FROM transactions WHERE address = ?",
(mempool_address, ))
debit_mempool = 0
if result:
for x in result:
debit_tx = quantize_eight(x[0])
fee = quantize_eight(
essentials.fee_calculate(
x[1], x[2], last_block))
debit_mempool = quantize_eight(debit_mempool +
debit_tx + fee)
credit = 0
for entry in essentials.execute_param_c(
c,
"SELECT amount FROM transactions WHERE recipient = ?",
(mempool_address, ), self.app_log):
credit = quantize_eight(credit) + quantize_eight(
entry[0])
debit_ledger = 0
for entry in essentials.execute_param_c(
c,
"SELECT amount FROM transactions WHERE address = ?",
(mempool_address, ), self.app_log):
debit_ledger = quantize_eight(
debit_ledger) + quantize_eight(entry[0])
debit = debit_ledger + debit_mempool
fees = 0
for entry in essentials.execute_param_c(
c,
"SELECT fee FROM transactions WHERE address = ?",
(mempool_address, ), self.app_log):
fees = quantize_eight(fees) + quantize_eight(
entry[0])
rewards = 0
for entry in essentials.execute_param_c(
c,
"SELECT sum(reward) FROM transactions WHERE recipient = ?",
(mempool_address, ), self.app_log):
rewards = quantize_eight(rewards) + quantize_eight(
entry[0])
# error conversion from NoneType to Decimal is not supported
balance = quantize_eight(
credit - debit - fees + rewards -
quantize_eight(mempool_amount))
balance_pre = quantize_eight(credit - debit_ledger -
fees + rewards)
fee = essentials.fee_calculate(mempool_openfield,
mempool_operation,
last_block)
# print("Balance", balance, fee)
if quantize_eight(mempool_amount) > quantize_eight(
balance_pre):
# mp amount is already included in "balance" var! also, that tx might already be in the mempool
mempool_result.append(
"Mempool: Sending more than owned")
continue
if quantize_eight(balance) - quantize_eight(fee) < 0:
mempool_result.append(
"Mempool: Cannot afford to pay fees")
continue
# Pfew! we can finally insert into mempool - all is str, type converted and enforced above
self.execute(
"INSERT INTO transactions VALUES (?,?,?,?,?,?,?,?,?)",
(mempool_timestamp, mempool_address,
mempool_recipient, mempool_amount,
mempool_signature_enc,
mempool_public_key_b64encoded, mempool_operation,
mempool_openfield, int(time_now)))
mempool_result.append(
"Mempool updated with a received transaction from {}"
.format(peer_ip))
mempool_result.append(
"Success"
) # WARNING: Do not change string or case ever!
self.commit(
) # Save (commit) the changes to mempool db
mempool_size += sys.getsizeof(
str(transaction)) / 1000000.0
else:
mempool_result.append(
"Local mempool is already full for this tx type, skipping merging"
)
# self.app_log.warning("Local mempool is already full for this tx type, skipping merging")
# TEMP
# print("Mempool insert", mempool_result)
return mempool_result
# TODO: Here maybe commit() on c to release the write lock?
except Exception as e:
self.app_log.warning("Mempool: Error processing: {} {}".format(
data, e))
if self.config.debug:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(
exc_tb.tb_frame.f_code.co_filename)[1]
self.app_log.warning("{} {} {}".format(
exc_type, fname, exc_tb.tb_lineno))
mempool_result.append("Exception: {}".format(str(e)))
# if left there, means debug can *not* be used in production, or exception is not sent back to the client.
raise
return mempool_result