def downloadAllPublicAddressOf(wallet_id, local_file="", start_page=1, show_time=False): if local_file == "": local_file = wallet_id + ".csv" # if the user doesn't provide a file to store the key, the function will generate one total_page = __getKeyPageNum(wallet_id) total_time = 0 for i in range(start_page, total_page + 1): start = time.time() url = 'https://www.walletexplorer.com/wallet/' + wallet_id + '/addresses?page=' + str( i) data = NetIO.readDataFrom(url) flag = 0 pattern2 = re.compile(r'<tr><td><a href=[^>]+') match = True while match: match = pattern2.search(data, flag) if match: flag = int(match.span()[-1]) sub = match.group()[26:-1] CsvIO.appendToFile(local_file, sub) else: break finish = time.time() t = finish - start total_time += t expect_left = (total_page - i) * t if show_time: print(str(i), 'tooks ', t, "secs") print(total_page, "time left:", expect_left / 60, "mins")
def getAllWallet(Wtype="Exchanges"): url = "https://www.walletexplorer.com/" result = [] data = NetIO.readDataFrom(url) # print(data) info = find_between(data, "<h3>" + Wtype, "</ul>") match = True pattern = re.compile(r'"/wallet/[^>]+') flag = 0 while match: match = pattern.search(info, flag) if (match): result.append(match.group()[9:-1]) # print(match.group()) flag = int(match.span()[-1]) else: break return result # print(findWalletByAddre("12t9YDPgwueZ9NyMgw519p7AA8isjr6SMw")) # print(findWalletByAddre("13AM4VW2dhxYgXeQepoHkHSQuy6NgaEb94")) # # print(findWalletByAddre("115p7UMMngoj1pMvkpHijcRdfJNXj6LrLn")) # result = transactionWalletQuery("d238e1e3533ee75e55b64ba1a5a0cf73a9fb87150ac97621a72d300d3ff157c7") # print({result["txid"]:result})
def downloadTransactionBetweenTime(wallet_id, end_time, start_time, store_path=""): # [start_time,end_time] # for easily update, accuracy to date end = 2017-10-26 start = 2017-02-21. The path should be a directory instead of a file total_page = __getPageNum(wallet_id) page = range(1, total_page + 1) find_end = False if (store_path == ""): store_path = wallet_id + start_time + "To" + end_time if not os.path.exists(store_path): os.makedirs(store_path) for i in page: url = 'https://www.walletexplorer.com/wallet/' + wallet_id + '?page=' + str( i) + '&format=csv' local_file = store_path + "/" + wallet_id + str(i) + '.csv' data = NetIO.readDataFrom(url) end, start = __findTime(data) if (end >= start_time and end <= end_time) or (start >= start_time and start <= end_time): CsvIO.writeToFile(local_file, data) elif (end < start_time): break
def unspentOutput(address, limit=250, confirmNum=6): # Multiple Addresses Allowed separated by "|" , Address can be base58 or xpub if limit > 1000: limit = 1000 url = "https://blockchain.info/unspent?active=" + address + "&limit=" + str( limit) + "&confirmations=" + str(confirmNum) return str2Object(NetIO.readDataFrom(url))
def findWalletByAddre(address): url = 'https://www.walletexplorer.com/address/' + address data = NetIO.readDataFrom(url) leading = 'part of wallet <a href="/wallet/' pattern = re.compile(r'part of wallet <a href="/wallet/[^>]+') match = re.search(pattern, data) addre = match.group() #print(addre) return addre[len(leading):-1]
def __getPageNum(wallet_id): url = 'https://www.walletexplorer.com/wallet/' + wallet_id data = NetIO.readDataFrom(url) pattern = re.compile(r'Page 1 / [0-9]+') match = re.search(pattern, data) num = match.group() num = int(num[9:]) return num
def downloadAllPublicAddressOf(wallet_id, local_file="", start_page=1, show_time=False, clear_file=False): if local_file == "": local_file = wallet_id + ".csv" # if the user doesn't provide a file to store the key, the function will generate one if (clear_file): CsvIO.writeToFile(local_file, "") #Clear the file first total_page = __getKeyPageNum(wallet_id) #total_time = 0; pattern2 = re.compile(r'<tr><td><a href=[^>]+') pattern_balance = re.compile(r'amount">[^&<]+') pattern_incomTx = re.compile(r"<td>[0-9]+") pattern_incomBlock = re.compile(r"<td>[0-9]+") for i in range(start_page, total_page + 1): pk_progress_moniter[wallet_id] = (i - 1) / total_page #start = time.time() url = 'https://www.walletexplorer.com/wallet/' + wallet_id + '/addresses?page=' + str( i) data = NetIO.readDataFrom(url) flag = 0 match = True while match: match = pattern2.search(data, flag) if match: flag = int(match.span()[-1]) sub = match.group()[26:-1] match_balance = pattern_balance.search(data, flag) flag = int(match_balance.span()[-1]) balance = match_balance.group()[8:] match_incomTx = pattern_incomTx.search(data, flag) flag = int(match_incomTx.span()[-1]) incomTx = match_incomTx.group()[4:] match_block = pattern_incomBlock.search(data, flag) flag = int(match_block.span()[-1]) blockID = match_block.group()[4:] #print(sub+","+balance+","+incomTx+","+blockID) CsvIO.appendToFile( local_file, sub + "," + balance + "," + incomTx + "," + blockID) else: break pk_progress_moniter[wallet_id] = (i) / total_page
def downloadTransactionBetweenTime(wallet_id, end_time, start_time, store_path="", download_transaction_detail=True, show_time=False): # [start_time,end_time] # for easily update, accuracy to date end = 2017-10-26 start = 2017-02-21. The path should be a directory instead of a file total_page = __getPageNum(wallet_id) page = range(1, total_page + 1) #find_end = False if (store_path == ""): store_path = wallet_id + "_" + start_time + "To" + end_time if not os.path.exists(store_path): os.makedirs(store_path) if (download_transaction_detail): append_file = store_path + "/Txdetail" if not os.path.exists(append_file): os.makedirs(append_file) #total_time = 0; for i in page: # tx_progress_moniter[wallet_id] = (i - 1) / total_page detail_file = append_file + "/transactionDetail_" + str(i) + ".json" #start = time.time() url = 'https://www.walletexplorer.com/wallet/' + wallet_id + '?page=' + str( i) + '&format=csv' local_file = store_path + "/" + wallet_id + "_" + str(i) + '.csv' data = NetIO.readDataFrom(url) end, start = __findTime(data) if (end >= start_time and end <= end_time) or (start >= start_time and start <= end_time): CsvIO.writeToFile(local_file, data) elif (end < start_time): break if (download_transaction_detail): # Also go through the transaction id on every page and append the json to a file d_data = CsvIO.readFile(local_file) dd = d_data.split("\n") all_txid = [] for ii in range(2, len(dd) - 1): all_txid.append(dd[ii].split(",")[-1].replace('"', "")) for txid in all_txid: #bottle neck here try: CsvIO.appendToFile(detail_file, str(tq(txid))) except: pass #boost a lot # thrs = [threading.Thread(target=__appendDetailFile, args=[detail_file,txid]) for txid in all_txid] # [thr.start() for thr in thrs] # [thr.join() for thr in thrs] tx_progress_moniter[wallet_id] = i / total_page
def downloadOneUrlPk(url, pattern2, pattern_balance, pattern_incomTx, pattern_incomBlock): global buffer, thread, error_log, error_url, log_file, store_path, url_file, all_wallet, result_file if (url == ""): return try: data = NetIO.readDataFrom(url) local_flag = 0 match = True while match: match = pattern2.search(data, local_flag) if match: local_flag = int(match.span()[-1]) sub = match.group()[26:-1] match_balance = pattern_balance.search(data, local_flag) local_flag = int(match_balance.span()[-1]) balance = match_balance.group()[8:] match_incomTx = pattern_incomTx.search(data, local_flag) local_flag = int(match_incomTx.span()[-1]) incomTx = match_incomTx.group()[4:] match_block = pattern_incomBlock.search(data, local_flag) local_flag = int(match_block.span()[-1]) blockID = match_block.group()[4:] # print(sub+","+balance+","+incomTx+","+blockID) buffer.append( util.find_between( url, "https://www.walletexplorer.com/wallet/", "/addresses") + "," + sub + "," + balance + "," + incomTx + "," + blockID) else: break except Exception as e: csv.appendToFile(error_log, str(e)) csv.appendToFile(error_url, url) #Call main to start #main()
def __getPublicAdr(wallet_id, page=1): url = 'https://www.walletexplorer.com/wallet/' + wallet_id + '/addresses?page=' + str( page) data = NetIO.readDataFrom(url) flag = 0 pattern2 = re.compile(r'<tr><td><a href=[^>]+') result = [] match = True while match: match = pattern2.search(data, flag) if match: flag = int(match.span()[-1]) sub = match.group()[26:-1] result.append(sub) else: break return result
def downloadOneCSV(url): global buffer, thread, error_log, error_url, log_file, store_path, url_file, all_wallet if (url == ""): return try: filename = url.replace("https://www.walletexplorer.com/wallet/", "").replace("?", "_").replace("format=csv", ".csv") content = NetIO.readDataFrom(url) csv.writeToFile(store_path + filename, content) except Exception as e: csv.appendToFile(error_log, str(e)) csv.appendToFile(error_url, url) #Call main to start #main()
def transactionQuery(tx_hash): url = "https://blockchain.info/rawtx/" + tx_hash return str2Object(NetIO.readDataFrom(url))
def blockHeight(block_height): url = "https://blockchain.info/block-height/" + str( block_height) + "?format=json" return str2Object(NetIO.readDataFrom(url))
def transactionWalletQuery(txid): result = {} rawdata = NetIO.readDataFrom("https://www.walletexplorer.com/txid/" + txid) soup = BeautifulSoup(rawdata, "lxml") # soup.find('table',class_="info") info = soup.find('table', class_="info").get_text() sender_wallet_id = soup.find('table', class_="info").find("a") try: sender_wallet_id = str(sender_wallet_id["href"])[8:] except: sender_wallet_id = "Multiple" #print(sender_wallet_id) # print(info) pattern_block = re.compile(r"block[0-9]+") pattern_time = re.compile(r"Time[^a-z]+") block = int(pattern_block.search(info).group()[5:]) block_col = int(find_between(info, "(pos ", ")")) time = pattern_time.search(info).group() fee = float(find_between(info, "Fee", "BTC")) size = int(find_between(info, "Size", " bytes")) # print(block,time,sender,fee) result["included_in_block"] = (block, block_col) date = datetime.strptime(time[4:-1], '%Y-%m-%d %H:%M:%S') #time_tool.mktime(date.timetuple()) result["time"] = int(time_tool.mktime(date.timetuple())) result["sender"] = sender_wallet_id result["fee"] = fee result["txid"] = txid result["size"] = size inout = soup.find_all('table', class_="empty") inAddress = inout[0] outAddress = inout[1] all_line = inAddress.find_all("tr") input_list = [] for line in all_line: cols = line.find_all("td") try: public_add = cols[0].find('a').get_text() except: public_add = "NONE" value = float(find_between(">" + cols[1].get_text(), ">", "BTC")) pre = find_between(str(cols[2]), "txid/", '">') input_list.append({ "publickey": public_add, "amount": value, "prev_tx_id": pre }) result["inputs"] = input_list all_line = outAddress.find_all("tr") my_set = [] for line in all_line: cols = line.find_all("td") try: public_add = cols[0].find('a').get_text() except: #print("Not valid transaction:",txid) public_add = "NONE" wallet_id = find_between(str(cols[1]), "wallet/", '">') value = float(find_between(">" + cols[2].get_text(), ">", "BTC")) next = find_between(str(cols[3]), "/txid/", '">') if next == "": next = "NONE" my_set.append({ "wallet_id": wallet_id, "publickey": public_add, "amount": value, "next_tx_id": next }) result["outputs"] = my_set return result
def blockQuery(block_hash): url = "https://blockchain.info/rawblock/" + block_hash return str2Object(NetIO.readDataFrom(url))
def addressesQuery(addresses): query = '' for add in addresses: query += add + "|" url = "https://blockchain.info/multiaddr?active=" + query[0:-1] return str2Object(NetIO.readDataFrom(url))
def addressQuery(address): url = "https://blockchain.info/rawaddr/" + address return str2Object(NetIO.readDataFrom(url))
def balanceQuery(address): # Multiple Addresses Allowed separated by "|" , Address can be base58 or xpub url = "https://blockchain.info/balance?active=" + address #return NetIO.readDataFrom(url) return str2Object(NetIO.readDataFrom(url))