class BKCoinTxCollector(CoinTxCollector):
std_offline_abi = [
"dumpData", "owner", "owner_assets", "sell_orders", "sell_orders_num",
"state"
]
def __init__(self, db):
super(BKCoinTxCollector, self).__init__()
self.stop_flag = False
self.db = db
self.order_list = []
self.config = BKCollectorConfig()
conf = {"host": self.config.RPC_HOST, "port": self.config.RPC_PORT}
self.wallet_api = WalletApi(self.config.ASSET_SYMBOL, conf)
def do_collect_app(self):
while self.stop_flag is False:
self.collect_token_contract()
time.sleep(10)
return ""
def collect_token_contract(self):
ret = self.wallet_api.http_request("get_contract_storage_changed", [0])
if not ret.has_key('result') or ret['result'] == None:
logging.info("Get contract failed")
return
for c in ret["result"]:
if self._check_contract_type(c["contract_address"]):
self._get_token_contract_info(c["contract_address"],
c["block_num"])
if len(self.order_list) > 0:
self.db.b_exchange_contracts.insert_many(self.order_list,
ordered=False)
self.order_list = []
def _check_contract_type(self, contract_address):
ret = self.wallet_api.http_request("get_contract_info",
[contract_address])
if ret['result'] == None:
logging.info("Call get_contract_info error")
return False
offline_abi = ret['result']['code_printable']['offline_abi']
for abi in BKCoinTxCollector.std_offline_abi:
if offline_abi.index(abi) >= 0:
logging.debug(abi)
else:
logging.info("Not standard token contract: " + abi)
return False
ret = self.wallet_api.http_request(
"invoke_contract_offline",
[self.config.CONTRACT_CALLER, contract_address, "state", ""])
if ret.has_key('result') and ret['result'] == "COMMON":
logging.debug("Contract state is good: " + contract_address +
" | " + ret['result'])
return True
else:
logging.info("Contract state error: " + contract_address + " | " +
ret['result'])
return False
def _get_token_contract_info(self, contract_address, block_num):
ret = self.wallet_api.http_request(
"invoke_contract_offline",
[self.config.CONTRACT_CALLER, contract_address, "sell_orders", ""])
if ret['result'] == None:
logging.info("get_contract_order error")
return
result = json.loads(ret['result'])
if isinstance(result, dict):
for k, v in result.items():
[from_asset, to_asset] = k.split(',')
order_info = json.loads(v)
for o in order_info['orderArray']:
[from_supply, to_supply, price] = o.split(',')
self.order_list.append({
"from_asset": from_asset,
"to_asset": to_asset,
"from_supply": from_supply,
"to_supply": to_supply,
"price": price,
"contract_address": contract_address,
"block_num": block_num
})
self.db.b_exchange_contracts.remove(
{"contract_address": contract_address})
class BTCCoinTxCollector(CoinTxCollector):
sync_status = True
def __init__(self, db):
super(BTCCoinTxCollector, self).__init__()
self.stop_flag = False
self.db = db
self.t_multisig_address = self.db.b_btc_multisig_address
self.multisig_address_cache = set()
self.config = BTCCollectorConfig()
conf = {"host": self.config.RPC_HOST, "port": self.config.RPC_PORT}
self.wallet_api = WalletApi(self.config.ASSET_SYMBOL, conf)
self.cache = CacheManager(self.config.SYNC_BLOCK_NUM,
self.config.ASSET_SYMBOL)
def _update_cache(self):
for addr in self.t_multisig_address.find({"addr_type": 0}):
self.multisig_address_cache.add(addr["address"])
def do_collect_app(self):
self._update_cache()
self.collect_thread = CollectBlockThread(self.db, self.config,
self.wallet_api,
self.sync_status)
self.collect_thread.start()
count = 0
last_block = 0
while self.stop_flag is False:
count += 1
block = q.get()
if last_block >= block.block_num:
logging.error("Unordered block number: " + str(last_block) +
":" + str(block.block_num))
last_block = block.block_num
# Update block table
# t_block = self.db.b_block
logging.debug("Block number: " + str(block.block_num) +
", Transaction number: " +
str(len(block.transactions)))
self.cache.block_cache.append(block.get_json_data())
# Process each transaction
for trx_data in block.transactions:
logging.debug("Transaction: %s" % trx_data)
if self.config.ASSET_SYMBOL == "HC":
if block.block_num == 0:
continue
trx_data = self.get_transaction_data(trx_data)
pretty_trx_info = self.collect_pretty_transaction(
self.db, trx_data, block.block_num)
self.sync_status = self.collect_thread.get_sync_status()
if self.sync_status:
logging.debug(str(count) + " blocks processed, flush to db")
self.cache.flush_to_db(self.db)
elif self.sync_status is False:
self.cache.flush_to_db(self.db)
self._update_cache()
time.sleep(2)
self.collect_thread.stop()
self.collect_thread.join()
def get_transaction_data(self, trx_id):
ret = self.wallet_api.http_request("getrawtransaction", [trx_id, 1])
if ret["result"] is None:
resp_data = None
else:
resp_data = ret["result"]
return resp_data
def collect_pretty_transaction(self, db_pool, base_trx_data, block_num):
trx_data = {}
trx_data["chainId"] = self.config.ASSET_SYMBOL.lower()
trx_data["trxid"] = base_trx_data["txid"]
trx_data["blockNum"] = block_num
vin = base_trx_data["vin"]
vout = base_trx_data["vout"]
trx_data["vout"] = []
trx_data["vin"] = []
# is_coinbase_trx = self.is_coinbase_transaction(base_trx_data)
out_set = {}
in_set = {}
multisig_in_addr = ""
multisig_out_addr = ""
is_valid_tx = True
"""
Only 3 types of transactions will be filtered out and be record in database.
1. deposit transaction (vin contains only one no LINK address and vout contains only one LINK address)
2. withdraw transaction (vin contains only one LINK address and vout contains no other LINK address)
3. transaction between hot-wallet and cold-wallet (vin contains only one LINK address and vout contains only one other LINK address)
Check logic:
1. check all tx in vin and store addresses & values (if more than one LINK address set invalid)
2. check all tx in vout and store all non-change addresses & values (if more than one LINK address set invalid)
3. above logic filter out the situation - more than one LINK address in vin or vout but there is one condition
should be filter out - more than one normal address in vin for deposit transaction
4. then we can record the transaction according to transaction type
only one other addres in vin and only one LINK address in vout - deposit
only one LINK addres in vin and only other addresses in vout - withdraw
only one LINK addres in vin and only one other LINK address in vout - transaction between hot-wallet and cold-wallet
no LINK address in vin and no LINK address in vout - transaction that we don't care about, record nothing
5. record original transaction in raw table if we care about it.
"""
for trx_in in vin:
if not trx_in.has_key("txid"):
continue
if not trx_in.has_key('vout'):
logging.error(trx_in)
utxo_id = self._cal_UTXO_prefix(trx_in['txid'], trx_in['vout'])
in_trx = self.cache.get_utxo(utxo_id)
self.cache.spend_utxo(utxo_id)
if in_trx is None:
logging.info(
"Fail to get vin transaction [%s:%d] of [%s]" %
(trx_in["txid"], trx_in['vout'], trx_data["trxid"]))
if self.config.ASSET_SYMBOL == "HC":
ret1 = self.wallet_api.http_request(
"getrawtransaction", [trx_in['txid'], 2])
else:
ret1 = self.wallet_api.http_request(
"getrawtransaction", [trx_in['txid'], True])
if not ret1.has_key('result'):
logging.error(
"Fail to get vin transaction [%s:%d] of [%s]" %
(trx_in["txid"], trx_in['vout'], trx_data["trxid"]))
exit(0)
addr = self._get_vout_address(
ret1.get("result").get("vout")[int(trx_in['vout'])])
if addr == "":
continue
if self.cache.balance_spent.has_key(addr):
self.cache.balance_spent[addr].append(utxo_id)
else:
self.cache.balance_spent[addr] = [utxo_id]
for t in ret1['result']['vout']:
if t['n'] == trx_in['vout']:
in_trx = {
'address': self._get_vout_address(t),
'value': t['value']
}
break
in_address = in_trx["address"]
if (in_set.has_key(in_address)):
in_set[in_address] += in_trx["value"]
else:
in_set[in_address] = in_trx["value"]
trx_data["vin"].append({
"txid": trx_in["txid"],
"vout": trx_in["vout"],
"value": in_trx["value"],
"address": in_address
})
if in_address in self.multisig_address_cache:
if multisig_in_addr == "":
multisig_in_addr = in_address
else:
if multisig_in_addr != in_address:
is_valid_tx = False
for trx_out in vout:
# Update UBXO cache
if trx_out["scriptPubKey"]["type"] == "nonstandard":
self.cache.add_utxo(
self._cal_UTXO_prefix(base_trx_data["txid"], trx_out["n"]),
{
"address":
"",
"value":
0 if
(not trx_out.has_key("value")) else trx_out["value"]
})
elif trx_out["scriptPubKey"].has_key("addresses"):
address = self._get_vout_address(trx_out)
self.cache.add_utxo(
self._cal_UTXO_prefix(base_trx_data["txid"], trx_out["n"]),
{
"address":
address,
"value":
0 if
(not trx_out.has_key("value")) else trx_out["value"]
})
# Check vout
if trx_out["scriptPubKey"].has_key("addresses"):
out_address = trx_out["scriptPubKey"]["addresses"][0]
trx_data["vout"].append({
"value":
trx_out["value"],
"n":
trx_out["n"],
"scriptPubKey":
trx_out["scriptPubKey"]["hex"],
"address":
out_address
})
if in_set.has_key(out_address): # remove change
continue
if (out_set.has_key(out_address)):
out_set[out_address] += trx_out["value"]
else:
out_set[out_address] = trx_out["value"]
if out_address in self.multisig_address_cache:
if multisig_out_addr == "":
multisig_out_addr = out_address
else:
is_valid_tx = False
if not multisig_in_addr == "" and not multisig_out_addr == "": # maybe transfer between hot-wallet and cold-wallet
if not is_valid_tx:
logging.error(
"Invalid transaction between hot-wallet and cold-wallet")
trx_data['type'] = -3
else:
trx_data['type'] = 0
elif not multisig_in_addr == "": # maybe withdraw
if not is_valid_tx:
logging.error("Invalid withdraw transaction")
trx_data['type'] = -1
else:
trx_data['type'] = 1
elif not multisig_out_addr == "": # maybe deposit
if not is_valid_tx or not len(in_set) == 1:
logging.error("Invalid deposit transaction")
trx_data['type'] = -2
else:
trx_data['type'] = 2
else:
logging.debug("Nothing to record")
return
#trx_data["trxTime"] = datetime.utcfromtimestamp(base_trx_data['time']).strftime("%Y-%m-%d %H:%M:%S")
#trx_data["createtime"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
if trx_data['type'] == 2 or trx_data['type'] == 0:
deposit_data = {
"txid": base_trx_data["txid"],
"from_account": in_set.keys()[0],
"to_account": multisig_out_addr,
"amount": str(out_set.values()[0]),
"asset_symbol": self.config.ASSET_SYMBOL,
"blockNum": block_num,
"chainId": self.config.ASSET_SYMBOL.lower()
}
self.cache.deposit_transaction_cache.append(deposit_data)
elif trx_data['type'] == 1:
for k, v in out_set.items():
withdraw_data = {
"txid": base_trx_data["txid"],
"from_account": multisig_in_addr,
"to_account": k,
"amount": str(v),
"asset_symbol": self.config.ASSET_SYMBOL,
"blockNum": block_num,
"chainId": self.config.ASSET_SYMBOL.lower()
}
self.cache.withdraw_transaction_cache.append(withdraw_data)
# logging.info("add raw transaction")
self.cache.raw_transaction_cache.append(trx_data)
return trx_data
def _cal_UTXO_prefix(self, txid, vout):
return self.config.ASSET_SYMBOL + txid + "I" + str(vout)
def _get_vout_address(self, vout_data):
if vout_data["scriptPubKey"]["type"] == "multisig":
address = pybitcointools.bin_to_b58check(
pybitcointools.hash160(vout_data["scriptPubKey"]["hex"]),
self.config.MULTISIG_VERSION)
elif vout_data["scriptPubKey"][
"type"] == "witness_v0_scripthash" or vout_data[
"scriptPubKey"]["type"] == "witness_v0_keyhash":
address = vout_data["scriptPubKey"]["hex"]
else:
if not vout_data["scriptPubKey"].has_key("addresses"):
#ToDo: OP_ADD and other OP_CODE may add exectuing function
return ""
elif len(vout_data["scriptPubKey"]["addresses"]) > 1:
logging.error("error data: ", vout_data)
pass
address = vout_data["scriptPubKey"]["addresses"][0]
return address
@staticmethod
def _is_coinbase_transaction(trx_data):
if len(trx_data["vin"]) == 1 and trx_data["vin"][0].has_key(
"coinbase"):
return True
return False
class USDTCoinTxCollector(CoinTxCollector):
sync_status = True
def __init__(self, db):
super(USDTCoinTxCollector, self).__init__()
self.stop_flag = False
self.db = db
self.t_multisig_address = self.db.b_usdt_multisig_address
self.multisig_address_cache = set()
self.config = USDTCollectorConfig()
conf = {
"host": self.config.RPC_HOST,
"port": self.config.RPC_PORT,
"rpc_user": self.config.RPC_USER,
"rpc_password": self.config.RPC_PASSWORD
}
self.wallet_api = WalletApi(self.config.ASSET_SYMBOL, conf)
self.cache = CacheManager(self.config.SYNC_BLOCK_NUM,
self.config.ASSET_SYMBOL)
def _update_cache(self):
for addr in self.t_multisig_address.find({"addr_type": 0}):
self.multisig_address_cache.add(addr["address"])
def do_collect_app(self):
self._update_cache()
self.collect_thread = CollectBlockThread(self.db, self.config,
self.wallet_api,
self.sync_status)
self.collect_thread.start()
count = 0
last_block = 0
while self.stop_flag is False:
count += 1
block = q.get()
if last_block >= block.block_num:
logging.error("Unordered block number: " + str(last_block) +
":" + str(block.block_num))
last_block = block.block_num
# Update block table
# t_block = self.db.b_block
logging.debug("Block number: " + str(block.block_num) +
", Transaction number: " +
str(len(block.transactions)))
self.cache.block_cache.append(block.get_json_data())
# Process each transaction
#print block.block_num,block.transactions
for trx_id in block.transactions:
logging.debug("Transaction: %s" % trx_id)
if self.config.ASSET_SYMBOL == "USDT":
if block.block_num == 0:
continue
trx_data = self.get_transaction_data(trx_id)
pretty_trx_info = self.collect_pretty_transaction(
self.db, trx_data, block.block_num)
self.sync_status = self.collect_thread.get_sync_status()
if self.sync_status:
logging.debug(str(count) + " blocks processed, flush to db")
self.cache.flush_to_db(self.db)
elif self.sync_status is False:
self.cache.flush_to_db(self.db)
self._update_cache()
time.sleep(2)
self.collect_thread.stop()
self.collect_thread.join()
def get_transaction_data(self, trx_id):
ret = self.wallet_api.http_request("omni_gettransaction", [trx_id])
if ret["result"] is None:
resp_data = None
else:
resp_data = ret["result"]
return resp_data
def collect_pretty_transaction(self, db_pool, base_trx_data, block_num):
trx_data = {}
trx_data["chainId"] = self.config.ASSET_SYMBOL.lower()
trx_data["trxid"] = base_trx_data["txid"]
trx_data["blockNum"] = block_num
target_property_id = self.config.PROPERTYID
print trx_data["trxid"]
if not base_trx_data.has_key("type_int"):
print "not has type", base_trx_data
return trx_data
trx_type = base_trx_data["type_int"]
if trx_type == 0:
print base_trx_data
value = float(base_trx_data["amount"])
if not base_trx_data.has_key("propertyid"):
print "is create omni transaction:", base_trx_data
return trx_data
propertyId = base_trx_data["propertyid"]
elif trx_type == 4:
if not base_trx_data.has_key("ecosystem"):
print "not has ecosystem", base_trx_data
return trx_data
ecoSystem = base_trx_data["ecosystem"]
if ecoSystem != "main":
print "isn't main net transaction"
return trx_data
print base_trx_data
propertyId = -1
if not base_trx_data.has_key("subsends"):
print "not has subsends", base_trx_data
return trx_data
for data in base_trx_data["subsends"]:
if data["propertyid"] == target_property_id:
propertyId = data["propertyid"]
value = float(data["amount"])
else:
is_valid_tx = False
return trx_data
if propertyId != target_property_id:
is_valid_tx = False
return trx_data
from_address = base_trx_data["sendingaddress"]
to_address = base_trx_data["referenceaddress"]
is_valid_tx = base_trx_data["valid"]
fee = float(base_trx_data["fee"])
out_set = {}
in_set = {}
multisig_in_addr = ""
multisig_out_addr = ""
out_set[to_address] = value
in_set[from_address] = value
"""
Only 3 types of transactions will be filtered out and be record in database.
1. deposit transaction (from_address only one no LINK address and to_address only one LINK address)
2. withdraw transaction (from_address only one LINK address and to_address no other LINK address)
3. transaction between hot-wallet and cold-wallet (from_address contains only one LINK address and to_address contains only one other LINK address)
Check logic:
1. check all tx in vin and store addresses & values (if more than one LINK address set invalid)
2. check all tx in vout and store all non-change addresses & values (if more than one LINK address set invalid)
3. above logic filter out the situation - more than one LINK address in vin or vout but there is one condition
should be filter out - more than one normal address in vin for deposit transaction
4. then we can record the transaction according to transaction type
only one other addres in vin and only one LINK address in vout - deposit
only one LINK addres in vin and only other addresses in vout - withdraw
only one LINK addres in vin and only one other LINK address in vout - transaction between hot-wallet and cold-wallet
no LINK address in vin and no LINK address in vout - transaction that we don't care about, record nothing
5. record original transaction in raw table if we care about it.
"""
if is_valid_tx:
print "from_addrsss:", from_address, "to_address", to_address, "value", value, "type", trx_type, "propertyId", propertyId, "fee", fee
if from_address in self.multisig_address_cache:
if multisig_in_addr == "":
multisig_in_addr = from_address
if to_address in self.multisig_address_cache:
if multisig_out_addr == "":
multisig_out_addr = to_address
if not multisig_in_addr == "" and not multisig_out_addr == "": # maybe transfer between hot-wallet and cold-wallet
if not is_valid_tx:
logging.error(
"Invalid transaction between hot-wallet and cold-wallet")
trx_data['type'] = -3
else:
trx_data['type'] = 0
elif not multisig_in_addr == "": # maybe withdraw
if not is_valid_tx:
logging.error("Invalid withdraw transaction")
trx_data['type'] = -1
else:
trx_data['type'] = 1
elif not multisig_out_addr == "": # maybe deposit
if not is_valid_tx or not len(in_set) == 1:
logging.error("Invalid deposit transaction")
trx_data['type'] = -2
else:
trx_data['type'] = 2
else:
logging.debug("Nothing to record")
return
#trx_data["trxTime"] = datetime.utcfromtimestamp(base_trx_data['time']).strftime("%Y-%m-%d %H:%M:%S")
#trx_data["createtime"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
if trx_data['type'] == 2 or trx_data['type'] == 0:
deposit_data = {
"txid": base_trx_data["txid"],
"from_account": from_address,
"to_account": multisig_out_addr,
"amount": "%.8f" % value,
"fee": "%.8f" % fee,
"asset_symbol": self.config.ASSET_SYMBOL,
"blockNum": block_num,
"chainId": self.config.ASSET_SYMBOL.lower()
}
self.cache.deposit_transaction_cache.append(deposit_data)
elif trx_data['type'] == 1:
for k, v in out_set.items():
withdraw_data = {
"txid": base_trx_data["txid"],
"from_account": multisig_in_addr,
"to_account": k,
"amount": "%.8f" % value,
"fee": "%.8f" % fee,
"asset_symbol": self.config.ASSET_SYMBOL,
"blockNum": block_num,
"chainId": self.config.ASSET_SYMBOL.lower()
}
self.cache.withdraw_transaction_cache.append(withdraw_data)
# logging.info("add raw transaction")
#self.cache.raw_transaction_cache.append(trx_data)
return trx_data