def update_KMD_notarisations(unrecorded_KMD_txids):
logger.info(f"Updating KMD {len(unrecorded_KMD_txids)} notarisations...")
i = 0
start = time.time()
notary_last_ntx = get_notary_last_ntx()
num_unrecorded_KMD_txids = len(unrecorded_KMD_txids)
for txid in unrecorded_KMD_txids:
row_data = get_notarised_data(txid)
i += 1
if row_data is not None: # ignore TXIDs that are not notarisations
chain = row_data[0]
ntx_row = notarised_row()
ntx_row.chain = chain
ntx_row.block_height = row_data[1]
ntx_row.block_time = row_data[2]
ntx_row.block_datetime = row_data[3]
ntx_row.block_hash = row_data[4]
ntx_row.notaries = row_data[5]
ntx_row.notary_addresses = row_data[6]
ntx_row.ac_ntx_blockhash = row_data[7]
ntx_row.ac_ntx_height = row_data[8]
ntx_row.txid = row_data[9]
ntx_row.opret = row_data[10]
ntx_row.season = row_data[11]
ntx_row.score_value = get_chain_epoch_score_at(
ntx_row.season, ntx_row.server, ntx_row.chain,
int(ntx_row.block_time))
ntx_row.epoch = get_chain_epoch_at(ntx_row.season, ntx_row.server,
ntx_row.chain,
int(ntx_row.block_time))
if chain == "GLEEC":
ntx_row.server = get_gleec_ntx_server(ntx_row.txid)
if ntx_row.server == "Third_Party":
ntx_row.chain == "GLEEC-OLD"
else:
ntx_row.server = row_data[12]
if ntx_row.score_value > 0:
ntx_row.scored = True
else:
ntx_row.scored = False
ntx_row.update()
runtime = int(time.time() - start)
try:
pct = round(i / num_unrecorded_KMD_txids * 100, 3)
est_end = int(100 / pct * runtime)
logger.info(
str(pct) + "% :" + str(i) + "/" +
str(num_unrecorded_KMD_txids) + " records added to db [" +
str(runtime) + "/" + str(est_end) + " sec]")
except:
pass
logger.info("Notarised blocks updated!")
def import_ntx(season, server, chain):
existing_notarised_txids = get_existing_notarised_txids(
chain, season, server)
logger.info(f"existing_notarised_txids: {len(existing_notarised_txids)}")
import_txids_url = f"{OTHER_SERVER}/api/info/notarisation_txid_list/?season={season}&server={server}&chain={chain}"
import_txids = requests.get(import_txids_url).json()["results"]
logger.info(f"import_txids: {len(import_txids)}")
new_txids = list(set(import_txids) - set(existing_notarised_txids))
logger.info(f"new_txids: {len(new_txids)}")
logger.info(f"Processing ETA: {0.03*len(new_txids)} sec")
time.sleep(0.02)
j = 0
for txid in new_txids:
j += 1
logger.info(f">>> Importing {txid} {j}/{len(new_txids)}")
txid_url = f"{OTHER_SERVER}/api/info/notarised_txid/?txid={txid}"
time.sleep(0.02)
r = requests.get(txid_url)
try:
txid_info_resp = r.json()["results"]
for txid_info in txid_info_resp:
ntx_row = notarised_row()
ntx_row.chain = txid_info["chain"]
ntx_row.block_height = txid_info["block_height"]
ntx_row.block_time = txid_info["block_time"]
ntx_row.block_datetime = txid_info["block_datetime"]
ntx_row.block_hash = txid_info["block_hash"]
ntx_row.notaries = txid_info["notaries"]
ntx_row.ac_ntx_blockhash = txid_info["ac_ntx_blockhash"]
ntx_row.ac_ntx_height = txid_info["ac_ntx_height"]
ntx_row.txid = txid_info["txid"]
ntx_row.opret = txid_info["opret"]
ntx_row.epoch = txid_info["epoch"]
if len(txid_info["notary_addresses"]) == 0:
if ntx_row.chain == "BTC":
url = f"{THIS_SERVER}/api/info/notary_btc_txid/?txid={txid}"
local_info = requests.get(url).json()["results"]
local_addresses = []
for item in local_info:
if item["input_index"] != -1:
local_addresses.append(item["address"])
ntx_row.notary_addresses = local_addresses
ntx_row.season, ntx_row.server = get_season_from_addresses(
ntx_row.notary_addresses, ntx_row.block_time,
"BTC", "BTC", txid, ntx_row.notaries)
elif ntx_row.chain == "LTC":
url = f"{THIS_SERVER}/api/info/notary_ltc_txid/?txid={txid}"
local_info = requests.get(url).json()["results"]
local_addresses = []
for item in local_info:
if item["input_index"] != -1:
local_addresses.append(item["address"])
ntx_row.notary_addresses = local_addresses
ntx_row.season, ntx_row.server = get_season_from_addresses(
ntx_row.notary_addresses, ntx_row.block_time,
"LTC", "LTC", txid, ntx_row.notaries)
else:
row_data = get_notarised_data(txid)
ntx_row.notary_addresses = row_data[6]
ntx_row.season = row_data[11]
ntx_row.server = row_data[12]
ntx_row.season, ntx_row.server = get_season_from_addresses(
ntx_row.notary_addresses, ntx_row.block_time,
"KMD", ntx_row.chain, txid, ntx_row.notaries)
else:
ntx_row.notary_addresses = txid_info["notary_addresses"]
ntx_row.season = txid_info["season"]
ntx_row.server = txid_info["server"]
ntx_row.score_value = get_chain_epoch_score_at(
ntx_row.season, ntx_row.server, ntx_row.chain,
int(ntx_row.block_time))
ntx_row.epoch = get_chain_epoch_at(ntx_row.season,
ntx_row.server,
ntx_row.chain,
int(ntx_row.block_time))
if ntx_row.chain == "GLEEC":
ntx_row.server = get_gleec_ntx_server(ntx_row.txid)
if ntx_row.server == "Third_Party":
ntx_row.chain == "GLEEC-OLD"
if ntx_row.score_value > 0:
ntx_row.scored = True
else:
ntx_row.scored = False
ntx_row.update()
except Exception as e:
logger.error(e)
logger.error(f"Something wrong with API? {txid_url}")
def scan_btc_transactions(season):
season_btc_addresses = NOTARY_BTC_ADDRESSES[season][:] + [BTC_NTX_ADDR]
num_addr = len(season_btc_addresses)
notary_last_ntx = get_notary_last_ntx("BTC")
i = 0
while len(season_btc_addresses) > 0:
if BTC_NTX_ADDR in season_btc_addresses:
notary_address = BTC_NTX_ADDR
else:
notary_address = random.choice(season_btc_addresses)
i += 1
if notary_address in NN_BTC_ADDRESSES_DICT[season]:
notary_name = NN_BTC_ADDRESSES_DICT[season][notary_address]
else:
notary_name = "non-NN"
existing_nn_btc_txids = get_existing_nn_btc_txids(notary_address)
existing_notarised_txids = get_existing_notarised_txids("BTC")
existing_txids = list(
set(existing_nn_btc_txids) & set(existing_notarised_txids))
txids = get_new_nn_btc_txids(existing_txids, notary_address)
logger.info(
f"[scan_btc_transactions] {len(existing_txids)} EXIST IN DB FOR {notary_address} | {notary_name} {season} ({i}/{num_addr})"
)
logger.info(
f"[scan_btc_transactions] {len(txids)} NEW TXIDs TO PROCESS FOR {notary_address} | {notary_name} {season} ({i}/{num_addr})"
)
num_txids = len(txids)
j = 0
for txid in txids:
j += 1
# Get tx data from Blockcypher API
logger.info(
f"[scan_btc_transactions] >>> Processing txid {j}/{num_txids}")
tx_info = get_btc_tx_info(txid, True, True)
if 'error' in tx_info:
pass
elif 'fees' in tx_info:
btc_row = tx_row()
btc_row.txid = txid
btc_row.address = notary_address
btc_row.fees = tx_info['fees']
btc_row.num_inputs = tx_info['vin_sz']
btc_row.num_outputs = tx_info['vout_sz']
btc_row.block_hash = tx_info['block_hash']
btc_row.block_height = tx_info['block_height']
block_time_iso8601 = tx_info['confirmed']
parsed_time = dp.parse(block_time_iso8601)
btc_row.block_time = parsed_time.strftime('%s')
btc_row.block_datetime = dt.utcfromtimestamp(
int(btc_row.block_time))
addresses = tx_info['addresses']
btc_row.season, btc_row.server = get_season_from_addresses(
addresses[:], btc_row.block_time, "BTC", "BTC")
vouts = tx_info["outputs"]
vins = tx_info["inputs"]
# update_notary_linked_vins(vins)
# single row for memo.sv spam
if detect_spam(btc_row, addresses, vouts):
logger.info("[scan_btc_transactions] SPAM detected")
elif detect_cipi_faucet(btc_row, addresses, vins):
logger.info("[scan_btc_transactions] CIPI_FAUCET detected")
# Detect Split (single row only)
elif detect_split(btc_row, addresses):
logger.info("[scan_btc_transactions] SPLIT detected")
else:
if txid in MadMax_personal_top_up:
btc_row.category = "MadMax personal top up"
elif txid in BTC_NTX_ADDR_consolidate:
btc_row.category = "BTC_NTX_ADDR consolidate"
elif txid in previous_season_funds_transfer:
btc_row.category = "previous season funds transfer"
elif txid in team_incoming:
btc_row.category = "team_incoming"
elif txid in REPORTED:
btc_row.category = "REPORTED"
#elif txid in pungo_other:
# btc_row.category = "Other"
elif detect_ntx(vins, vouts, addresses):
btc_row.category = "NTX"
elif detect_replenish(vins, vouts):
btc_row.category = "Replenish"
elif detect_consolidate(
vins, vouts
) or txid in dragonhound_consolidate or txid in strob_consolidate or txid in webworker_2step_consolidate:
btc_row.category = "Consolidate"
elif detect_intra_notary(vins, vouts):
btc_row.category = "Intra-Notary"
else:
btc_row.category = "Other"
notary_list = []
notary_addresses = []
input_index = 0
for vin in vins:
btc_row.output_sats = -1
btc_row.output_index = -1
btc_row.input_sats = vin['output_value']
btc_row.input_index = input_index
btc_row.address = vin["addresses"][0]
btc_row.notary = get_notary_from_btc_address(
btc_row.address, btc_row.season)
btc_row.update()
input_index += 1
notary_list.append(btc_row.notary)
notary_addresses.append(btc_row.address)
output_index = 0
for vout in vouts:
if vout["addresses"] is not None:
btc_row.input_index = -1
btc_row.input_sats = -1
btc_row.address = vout["addresses"][0]
btc_row.notary = get_notary_from_btc_address(
btc_row.address, btc_row.season)
btc_row.output_sats = vout['value']
btc_row.output_index = output_index
btc_row.update()
output_index += 1
if btc_row.category == "NTX":
for vout in vouts:
if 'data_hex' in vout:
opret = vout['data_hex']
opret_url = f'{THIS_SERVER}/api/tools/decode_opreturn/?OP_RETURN={opret}'
r = requests.get(opret_url)
kmd_ntx_info = r.json()
ac_ntx_height = kmd_ntx_info['notarised_block']
ac_ntx_blockhash = kmd_ntx_info[
'notarised_blockhash']
# Update "notarised" table
ntx_row = notarised_row()
ntx_row.chain = 'BTC'
ntx_row.block_height = btc_row.block_height
ntx_row.block_time = int(btc_row.block_time)
ntx_row.block_datetime = btc_row.block_datetime
ntx_row.block_hash = btc_row.block_hash
ntx_row.notaries = notary_list
ntx_row.notary_addresses = notary_addresses
ntx_row.ac_ntx_blockhash = ac_ntx_blockhash
ntx_row.ac_ntx_height = ac_ntx_height
ntx_row.txid = btc_row.txid
ntx_row.opret = opret
ntx_row.season = btc_row.season
ntx_row.server = btc_row.server
ntx_row.score_value = get_chain_epoch_score_at(
ntx_row.season, ntx_row.server,
ntx_row.chain, int(ntx_row.block_time))
ntx_row.epoch = get_chain_epoch_at(
ntx_row.season, ntx_row.server,
ntx_row.chain, int(ntx_row.block_time))
if ntx_row.score_value > 0:
ntx_row.scored = True
else:
ntx_row.scored = False
ntx_row.update()
logger.info(
f"[scan_btc_transactions] TXID: {txid} ({btc_row.category})"
)
else:
logger.warning(
f"[scan_btc_transactions] Fees not in txinfo for {txid}! Likely unconfirmed..."
)
season_btc_addresses.remove(notary_address)