def downloadAllPublicAddressOf(wallet_id,
local_file="",
start_page=1,
show_time=False):
if local_file == "":
local_file = wallet_id + ".csv" # if the user doesn't provide a file to store the key, the function will generate one
total_page = __getKeyPageNum(wallet_id)
total_time = 0
for i in range(start_page, total_page + 1):
start = time.time()
url = 'https://www.walletexplorer.com/wallet/' + wallet_id + '/addresses?page=' + str(
i)
data = NetIO.readDataFrom(url)
flag = 0
pattern2 = re.compile(r'<tr><td><a href=[^>]+')
match = True
while match:
match = pattern2.search(data, flag)
if match:
flag = int(match.span()[-1])
sub = match.group()[26:-1]
CsvIO.appendToFile(local_file, sub)
else:
break
finish = time.time()
t = finish - start
total_time += t
expect_left = (total_page - i) * t
if show_time:
print(str(i), 'tooks ', t, "secs")
print(total_page, "time left:", expect_left / 60, "mins")
def getAllWallet(Wtype="Exchanges"):
url = "https://www.walletexplorer.com/"
result = []
data = NetIO.readDataFrom(url)
# print(data)
info = find_between(data, "<h3>" + Wtype, "</ul>")
match = True
pattern = re.compile(r'"/wallet/[^>]+')
flag = 0
while match:
match = pattern.search(info, flag)
if (match):
result.append(match.group()[9:-1])
# print(match.group())
flag = int(match.span()[-1])
else:
break
return result
# print(findWalletByAddre("12t9YDPgwueZ9NyMgw519p7AA8isjr6SMw"))
# print(findWalletByAddre("13AM4VW2dhxYgXeQepoHkHSQuy6NgaEb94"))
#
# print(findWalletByAddre("115p7UMMngoj1pMvkpHijcRdfJNXj6LrLn"))
# result = transactionWalletQuery("d238e1e3533ee75e55b64ba1a5a0cf73a9fb87150ac97621a72d300d3ff157c7")
# print({result["txid"]:result})
def downloadTransactionBetweenTime(wallet_id,
end_time,
start_time,
store_path=""): # [start_time,end_time]
# for easily update, accuracy to date end = 2017-10-26 start = 2017-02-21. The path should be a directory instead of a file
total_page = __getPageNum(wallet_id)
page = range(1, total_page + 1)
find_end = False
if (store_path == ""):
store_path = wallet_id + start_time + "To" + end_time
if not os.path.exists(store_path):
os.makedirs(store_path)
for i in page:
url = 'https://www.walletexplorer.com/wallet/' + wallet_id + '?page=' + str(
i) + '&format=csv'
local_file = store_path + "/" + wallet_id + str(i) + '.csv'
data = NetIO.readDataFrom(url)
end, start = __findTime(data)
if (end >= start_time and end <= end_time) or (start >= start_time
and start <= end_time):
CsvIO.writeToFile(local_file, data)
elif (end < start_time):
break
def unspentOutput(address, limit=250, confirmNum=6):
# Multiple Addresses Allowed separated by "|" , Address can be base58 or xpub
if limit > 1000:
limit = 1000
url = "https://blockchain.info/unspent?active=" + address + "&limit=" + str(
limit) + "&confirmations=" + str(confirmNum)
return str2Object(NetIO.readDataFrom(url))
def findWalletByAddre(address):
url = 'https://www.walletexplorer.com/address/' + address
data = NetIO.readDataFrom(url)
leading = 'part of wallet <a href="/wallet/'
pattern = re.compile(r'part of wallet <a href="/wallet/[^>]+')
match = re.search(pattern, data)
addre = match.group()
#print(addre)
return addre[len(leading):-1]
def __getPageNum(wallet_id):
url = 'https://www.walletexplorer.com/wallet/' + wallet_id
data = NetIO.readDataFrom(url)
pattern = re.compile(r'Page 1 / [0-9]+')
match = re.search(pattern, data)
num = match.group()
num = int(num[9:])
return num
def downloadAllPublicAddressOf(wallet_id,
local_file="",
start_page=1,
show_time=False,
clear_file=False):
if local_file == "":
local_file = wallet_id + ".csv" # if the user doesn't provide a file to store the key, the function will generate one
if (clear_file):
CsvIO.writeToFile(local_file, "") #Clear the file first
total_page = __getKeyPageNum(wallet_id)
#total_time = 0;
pattern2 = re.compile(r'<tr><td><a href=[^>]+')
pattern_balance = re.compile(r'amount">[^&<]+')
pattern_incomTx = re.compile(r"<td>[0-9]+")
pattern_incomBlock = re.compile(r"<td>[0-9]+")
for i in range(start_page, total_page + 1):
pk_progress_moniter[wallet_id] = (i - 1) / total_page
#start = time.time()
url = 'https://www.walletexplorer.com/wallet/' + wallet_id + '/addresses?page=' + str(
i)
data = NetIO.readDataFrom(url)
flag = 0
match = True
while match:
match = pattern2.search(data, flag)
if match:
flag = int(match.span()[-1])
sub = match.group()[26:-1]
match_balance = pattern_balance.search(data, flag)
flag = int(match_balance.span()[-1])
balance = match_balance.group()[8:]
match_incomTx = pattern_incomTx.search(data, flag)
flag = int(match_incomTx.span()[-1])
incomTx = match_incomTx.group()[4:]
match_block = pattern_incomBlock.search(data, flag)
flag = int(match_block.span()[-1])
blockID = match_block.group()[4:]
#print(sub+","+balance+","+incomTx+","+blockID)
CsvIO.appendToFile(
local_file,
sub + "," + balance + "," + incomTx + "," + blockID)
else:
break
pk_progress_moniter[wallet_id] = (i) / total_page
def downloadTransactionBetweenTime(wallet_id,
end_time,
start_time,
store_path="",
download_transaction_detail=True,
show_time=False): # [start_time,end_time]
# for easily update, accuracy to date end = 2017-10-26 start = 2017-02-21. The path should be a directory instead of a file
total_page = __getPageNum(wallet_id)
page = range(1, total_page + 1)
#find_end = False
if (store_path == ""):
store_path = wallet_id + "_" + start_time + "To" + end_time
if not os.path.exists(store_path):
os.makedirs(store_path)
if (download_transaction_detail):
append_file = store_path + "/Txdetail"
if not os.path.exists(append_file):
os.makedirs(append_file)
#total_time = 0;
for i in page: #
tx_progress_moniter[wallet_id] = (i - 1) / total_page
detail_file = append_file + "/transactionDetail_" + str(i) + ".json"
#start = time.time()
url = 'https://www.walletexplorer.com/wallet/' + wallet_id + '?page=' + str(
i) + '&format=csv'
local_file = store_path + "/" + wallet_id + "_" + str(i) + '.csv'
data = NetIO.readDataFrom(url)
end, start = __findTime(data)
if (end >= start_time and end <= end_time) or (start >= start_time
and start <= end_time):
CsvIO.writeToFile(local_file, data)
elif (end < start_time):
break
if (download_transaction_detail):
# Also go through the transaction id on every page and append the json to a file
d_data = CsvIO.readFile(local_file)
dd = d_data.split("\n")
all_txid = []
for ii in range(2, len(dd) - 1):
all_txid.append(dd[ii].split(",")[-1].replace('"', ""))
for txid in all_txid:
#bottle neck here
try:
CsvIO.appendToFile(detail_file, str(tq(txid)))
except:
pass
#boost a lot
# thrs = [threading.Thread(target=__appendDetailFile, args=[detail_file,txid]) for txid in all_txid]
# [thr.start() for thr in thrs]
# [thr.join() for thr in thrs]
tx_progress_moniter[wallet_id] = i / total_page
def downloadOneUrlPk(url, pattern2, pattern_balance, pattern_incomTx,
pattern_incomBlock):
global buffer, thread, error_log, error_url, log_file, store_path, url_file, all_wallet, result_file
if (url == ""):
return
try:
data = NetIO.readDataFrom(url)
local_flag = 0
match = True
while match:
match = pattern2.search(data, local_flag)
if match:
local_flag = int(match.span()[-1])
sub = match.group()[26:-1]
match_balance = pattern_balance.search(data, local_flag)
local_flag = int(match_balance.span()[-1])
balance = match_balance.group()[8:]
match_incomTx = pattern_incomTx.search(data, local_flag)
local_flag = int(match_incomTx.span()[-1])
incomTx = match_incomTx.group()[4:]
match_block = pattern_incomBlock.search(data, local_flag)
local_flag = int(match_block.span()[-1])
blockID = match_block.group()[4:]
# print(sub+","+balance+","+incomTx+","+blockID)
buffer.append(
util.find_between(
url, "https://www.walletexplorer.com/wallet/",
"/addresses") + "," + sub + "," + balance + "," +
incomTx + "," + blockID)
else:
break
except Exception as e:
csv.appendToFile(error_log, str(e))
csv.appendToFile(error_url, url)
#Call main to start
#main()
def __getPublicAdr(wallet_id, page=1):
url = 'https://www.walletexplorer.com/wallet/' + wallet_id + '/addresses?page=' + str(
page)
data = NetIO.readDataFrom(url)
flag = 0
pattern2 = re.compile(r'<tr><td><a href=[^>]+')
result = []
match = True
while match:
match = pattern2.search(data, flag)
if match:
flag = int(match.span()[-1])
sub = match.group()[26:-1]
result.append(sub)
else:
break
return result
def downloadOneCSV(url):
global buffer, thread, error_log, error_url, log_file, store_path, url_file, all_wallet
if (url == ""):
return
try:
filename = url.replace("https://www.walletexplorer.com/wallet/",
"").replace("?",
"_").replace("format=csv", ".csv")
content = NetIO.readDataFrom(url)
csv.writeToFile(store_path + filename, content)
except Exception as e:
csv.appendToFile(error_log, str(e))
csv.appendToFile(error_url, url)
#Call main to start
#main()
def transactionQuery(tx_hash):
url = "https://blockchain.info/rawtx/" + tx_hash
return str2Object(NetIO.readDataFrom(url))
def blockHeight(block_height):
url = "https://blockchain.info/block-height/" + str(
block_height) + "?format=json"
return str2Object(NetIO.readDataFrom(url))
def transactionWalletQuery(txid):
result = {}
rawdata = NetIO.readDataFrom("https://www.walletexplorer.com/txid/" + txid)
soup = BeautifulSoup(rawdata, "lxml")
# soup.find('table',class_="info")
info = soup.find('table', class_="info").get_text()
sender_wallet_id = soup.find('table', class_="info").find("a")
try:
sender_wallet_id = str(sender_wallet_id["href"])[8:]
except:
sender_wallet_id = "Multiple"
#print(sender_wallet_id)
# print(info)
pattern_block = re.compile(r"block[0-9]+")
pattern_time = re.compile(r"Time[^a-z]+")
block = int(pattern_block.search(info).group()[5:])
block_col = int(find_between(info, "(pos ", ")"))
time = pattern_time.search(info).group()
fee = float(find_between(info, "Fee", "BTC"))
size = int(find_between(info, "Size", " bytes"))
# print(block,time,sender,fee)
result["included_in_block"] = (block, block_col)
date = datetime.strptime(time[4:-1], '%Y-%m-%d %H:%M:%S')
#time_tool.mktime(date.timetuple())
result["time"] = int(time_tool.mktime(date.timetuple()))
result["sender"] = sender_wallet_id
result["fee"] = fee
result["txid"] = txid
result["size"] = size
inout = soup.find_all('table', class_="empty")
inAddress = inout[0]
outAddress = inout[1]
all_line = inAddress.find_all("tr")
input_list = []
for line in all_line:
cols = line.find_all("td")
try:
public_add = cols[0].find('a').get_text()
except:
public_add = "NONE"
value = float(find_between(">" + cols[1].get_text(), ">", "BTC"))
pre = find_between(str(cols[2]), "txid/", '">')
input_list.append({
"publickey": public_add,
"amount": value,
"prev_tx_id": pre
})
result["inputs"] = input_list
all_line = outAddress.find_all("tr")
my_set = []
for line in all_line:
cols = line.find_all("td")
try:
public_add = cols[0].find('a').get_text()
except:
#print("Not valid transaction:",txid)
public_add = "NONE"
wallet_id = find_between(str(cols[1]), "wallet/", '">')
value = float(find_between(">" + cols[2].get_text(), ">", "BTC"))
next = find_between(str(cols[3]), "/txid/", '">')
if next == "":
next = "NONE"
my_set.append({
"wallet_id": wallet_id,
"publickey": public_add,
"amount": value,
"next_tx_id": next
})
result["outputs"] = my_set
return result
def blockQuery(block_hash):
url = "https://blockchain.info/rawblock/" + block_hash
return str2Object(NetIO.readDataFrom(url))
def addressesQuery(addresses):
query = ''
for add in addresses:
query += add + "|"
url = "https://blockchain.info/multiaddr?active=" + query[0:-1]
return str2Object(NetIO.readDataFrom(url))
def addressQuery(address):
url = "https://blockchain.info/rawaddr/" + address
return str2Object(NetIO.readDataFrom(url))
def balanceQuery(address):
# Multiple Addresses Allowed separated by "|" , Address can be base58 or xpub
url = "https://blockchain.info/balance?active=" + address
#return NetIO.readDataFrom(url)
return str2Object(NetIO.readDataFrom(url))