def extractTxid(wallet_tx_path, txid_store_file):
print("Extracting")
f = open(txid_store_file, 'a', encoding="utf-8")
for filename in os.listdir(wallet_tx_path):
try:
content = csv.readFile(wallet_tx_path + filename)
content = content.split("\n")
all_txid = []
for ii in range(2, len(content) - 1):
all_txid.append(content[ii].split(",")[-1].replace('"', ""))
for id in all_txid:
f.write(id + '\n')
except:
continue
f.close()
print("Deleting duplication")
content = csv.readFile(txid_store_file)
content = content.split("\n")
clear_content = set(content)
try:
clear_content.remove("")
except:
pass
csv.writeToFile(txid_store_file, "") # delete original
f = open(txid_store_file, 'a', encoding="utf-8")
for i in clear_content:
f.write(i + "\n")
f.close()
print("Finish")
def downloadTransactionBetweenTime(wallet_id,
end_time,
start_time,
store_path=""): # [start_time,end_time]
# for easily update, accuracy to date end = 2017-10-26 start = 2017-02-21. The path should be a directory instead of a file
total_page = __getPageNum(wallet_id)
page = range(1, total_page + 1)
find_end = False
if (store_path == ""):
store_path = wallet_id + start_time + "To" + end_time
if not os.path.exists(store_path):
os.makedirs(store_path)
for i in page:
url = 'https://www.walletexplorer.com/wallet/' + wallet_id + '?page=' + str(
i) + '&format=csv'
local_file = store_path + "/" + wallet_id + str(i) + '.csv'
data = NetIO.readDataFrom(url)
end, start = __findTime(data)
if (end >= start_time and end <= end_time) or (start >= start_time
and start <= end_time):
CsvIO.writeToFile(local_file, data)
elif (end < start_time):
break
def downloadAllPublicAddressOf(wallet_id,
local_file="",
start_page=1,
show_time=False):
if local_file == "":
local_file = wallet_id + ".csv" # if the user doesn't provide a file to store the key, the function will generate one
total_page = __getKeyPageNum(wallet_id)
total_time = 0
for i in range(start_page, total_page + 1):
start = time.time()
url = 'https://www.walletexplorer.com/wallet/' + wallet_id + '/addresses?page=' + str(
i)
data = NetIO.readDataFrom(url)
flag = 0
pattern2 = re.compile(r'<tr><td><a href=[^>]+')
match = True
while match:
match = pattern2.search(data, flag)
if match:
flag = int(match.span()[-1])
sub = match.group()[26:-1]
CsvIO.appendToFile(local_file, sub)
else:
break
finish = time.time()
t = finish - start
total_time += t
expect_left = (total_page - i) * t
if show_time:
print(str(i), 'tooks ', t, "secs")
print(total_page, "time left:", expect_left / 60, "mins")
def downloadAllPublicAddressOf(wallet_id,
local_file="",
start_page=1,
show_time=False,
clear_file=False):
if local_file == "":
local_file = wallet_id + ".csv" # if the user doesn't provide a file to store the key, the function will generate one
if (clear_file):
CsvIO.writeToFile(local_file, "") #Clear the file first
total_page = __getKeyPageNum(wallet_id)
#total_time = 0;
pattern2 = re.compile(r'<tr><td><a href=[^>]+')
pattern_balance = re.compile(r'amount">[^&<]+')
pattern_incomTx = re.compile(r"<td>[0-9]+")
pattern_incomBlock = re.compile(r"<td>[0-9]+")
for i in range(start_page, total_page + 1):
pk_progress_moniter[wallet_id] = (i - 1) / total_page
#start = time.time()
url = 'https://www.walletexplorer.com/wallet/' + wallet_id + '/addresses?page=' + str(
i)
data = NetIO.readDataFrom(url)
flag = 0
match = True
while match:
match = pattern2.search(data, flag)
if match:
flag = int(match.span()[-1])
sub = match.group()[26:-1]
match_balance = pattern_balance.search(data, flag)
flag = int(match_balance.span()[-1])
balance = match_balance.group()[8:]
match_incomTx = pattern_incomTx.search(data, flag)
flag = int(match_incomTx.span()[-1])
incomTx = match_incomTx.group()[4:]
match_block = pattern_incomBlock.search(data, flag)
flag = int(match_block.span()[-1])
blockID = match_block.group()[4:]
#print(sub+","+balance+","+incomTx+","+blockID)
CsvIO.appendToFile(
local_file,
sub + "," + balance + "," + incomTx + "," + blockID)
else:
break
pk_progress_moniter[wallet_id] = (i) / total_page
def downloadTransactionBetweenTime(wallet_id,
end_time,
start_time,
store_path="",
download_transaction_detail=True,
show_time=False): # [start_time,end_time]
# for easily update, accuracy to date end = 2017-10-26 start = 2017-02-21. The path should be a directory instead of a file
total_page = __getPageNum(wallet_id)
page = range(1, total_page + 1)
#find_end = False
if (store_path == ""):
store_path = wallet_id + "_" + start_time + "To" + end_time
if not os.path.exists(store_path):
os.makedirs(store_path)
if (download_transaction_detail):
append_file = store_path + "/Txdetail"
if not os.path.exists(append_file):
os.makedirs(append_file)
#total_time = 0;
for i in page: #
tx_progress_moniter[wallet_id] = (i - 1) / total_page
detail_file = append_file + "/transactionDetail_" + str(i) + ".json"
#start = time.time()
url = 'https://www.walletexplorer.com/wallet/' + wallet_id + '?page=' + str(
i) + '&format=csv'
local_file = store_path + "/" + wallet_id + "_" + str(i) + '.csv'
data = NetIO.readDataFrom(url)
end, start = __findTime(data)
if (end >= start_time and end <= end_time) or (start >= start_time
and start <= end_time):
CsvIO.writeToFile(local_file, data)
elif (end < start_time):
break
if (download_transaction_detail):
# Also go through the transaction id on every page and append the json to a file
d_data = CsvIO.readFile(local_file)
dd = d_data.split("\n")
all_txid = []
for ii in range(2, len(dd) - 1):
all_txid.append(dd[ii].split(",")[-1].replace('"', ""))
for txid in all_txid:
#bottle neck here
try:
CsvIO.appendToFile(detail_file, str(tq(txid)))
except:
pass
#boost a lot
# thrs = [threading.Thread(target=__appendDetailFile, args=[detail_file,txid]) for txid in all_txid]
# [thr.start() for thr in thrs]
# [thr.join() for thr in thrs]
tx_progress_moniter[wallet_id] = i / total_page
def downloadOneTx(url):
global buffer, thread, error_log, error_url, log_file, store_path, url_file, error_num, result_file
if (url == ""):
return
try:
result = str(getBlockInfoTran(url))
buffer.append(result)
except Exception as e:
csv.appendToFile(error_log, str(e))
csv.appendToFile(error_url, url)
error_num += 1
def addMonitorFor(target_function, time_inter):
while (not target_function.finish):
remember = target_function.flag
time.sleep(time_inter)
if (remember == target_function.flag):
csv.appendToFile(
center_log,
str(time.time()) + ":" + str(target_function) +
" stopped, restoring...")
main_thred = threading.Thread(target=target_function.restore)
moniter = threading.Thread(target=addMonitorFor,
args=[target_function, time_inter])
main_thred.start()
moniter.start()
main_thred.join()
moniter.join()
def downloadOneUrlPk(url, pattern2, pattern_balance, pattern_incomTx,
pattern_incomBlock):
global buffer, thread, error_log, error_url, log_file, store_path, url_file, all_wallet, result_file
if (url == ""):
return
try:
data = NetIO.readDataFrom(url)
local_flag = 0
match = True
while match:
match = pattern2.search(data, local_flag)
if match:
local_flag = int(match.span()[-1])
sub = match.group()[26:-1]
match_balance = pattern_balance.search(data, local_flag)
local_flag = int(match_balance.span()[-1])
balance = match_balance.group()[8:]
match_incomTx = pattern_incomTx.search(data, local_flag)
local_flag = int(match_incomTx.span()[-1])
incomTx = match_incomTx.group()[4:]
match_block = pattern_incomBlock.search(data, local_flag)
local_flag = int(match_block.span()[-1])
blockID = match_block.group()[4:]
# print(sub+","+balance+","+incomTx+","+blockID)
buffer.append(
util.find_between(
url, "https://www.walletexplorer.com/wallet/",
"/addresses") + "," + sub + "," + balance + "," +
incomTx + "," + blockID)
else:
break
except Exception as e:
csv.appendToFile(error_log, str(e))
csv.appendToFile(error_url, url)
#Call main to start
#main()
def main():
global buffer, thread, error_log, error_url, log_file, store_path, url_file, all_wallet, result_file
init()
if not os.path.exists(store_path):
os.makedirs(store_path)
if not os.path.exists(log_path):
os.makedirs(log_path)
error_log = log_path + error_log
error_url = log_path + error_url
log_file = log_path + log_file
url_file = log_path + url_file
result_file = store_path + result_file
csv.writeToFile(url_file, "") # clear the file
downladPkFor(all_wallet)
def getHashpub2wallet(hashdir):
the_map = {}
lst = os.listdir(hashdir)
lst.sort()
for file in lst:
content = CsvIO.readFile(hashdir + "/" + file)
content = content.split("\n")
for line in content:
#print(line,file.replace(".csv",""))
the_map[line] = file.replace(
".csv", "") # this filename must be the wallet_id
#print("finish ",file)
return the_map
def downloadTxOnly(txidpath, deduplicate=True, hint=""):
if deduplicate:
content = csv.readFile(txidpath)
content = content.split("\n")
clear_content = set(content)
try:
clear_content.remove("")
except:
pass
csv.writeToFile(txidpath, "") # delete original
f = open(txidpath, 'a', encoding="utf-8")
for i in clear_content:
f.write(i + "\n")
f.close()
DetailedTransactionDownloader.init()
DetailedTransactionDownloader.url_file = txidpath
main_thred = threading.Thread(target=DetailedTransactionDownloader.main)
moniter = threading.Thread(target=addMonitorFor,
args=[DetailedTransactionDownloader, 30])
main_thred.start()
moniter.start()
main_thred.join()
moniter.join()
def downloadOneCSV(url):
global buffer, thread, error_log, error_url, log_file, store_path, url_file, all_wallet
if (url == ""):
return
try:
filename = url.replace("https://www.walletexplorer.com/wallet/",
"").replace("?",
"_").replace("format=csv", ".csv")
content = NetIO.readDataFrom(url)
csv.writeToFile(store_path + filename, content)
except Exception as e:
csv.appendToFile(error_log, str(e))
csv.appendToFile(error_url, url)
#Call main to start
#main()
def beginDownload():
global buffer, thread, error_log, error_url, log_file, store_path, url_file, result_file, flag, content, finish, total_task
content = csv.readFile(url_file)
content = content.split("\n")
start = flag
old_flag = start
end = len(content)
total_task = end
start_time = time.time()
while start + thread < end:
sub_content = content[start:start + thread]
del buffer[:]
try:
thrs = [
threading.Thread(target=downloadOneTx, args=[url])
for url in sub_content
]
[thr.start() for thr in thrs]
[thr.join() for thr in thrs]
except Exception as e:
csv.appendToFile(error_log, str(e))
csv.appendToFile(
error_log,
"Affected url:" + str(start) + " to " + str(start + thread))
pass
with open(result_file, "a", encoding="utf-8") as f:
for record in buffer:
f.write(record + "\n")
del buffer[:]
start += thread
flag = start
if ((end - old_flag) != 0):
progress = (start - old_flag) / (end - old_flag)
else:
progress = 1
total_time = time.time() - start_time
expected_time_left = total_time / progress - total_time
log_str = "Current Progress: " + str(
progress * 100) + "%, Already run:" + str(
total_time / 60) + "mins, Expected Left:" + str(
expected_time_left / 60) + "mins, flag:" + str(flag)
csv.appendToFile(log_file, log_str)
sub_content = content[start:end]
del buffer[:]
try:
thrs = [
threading.Thread(target=downloadOneTx, args=[url])
for url in sub_content
]
[thr.start() for thr in thrs]
[thr.join() for thr in thrs]
except Exception as e:
csv.appendToFile(error_log, str(e))
csv.appendToFile(
error_log,
"Affected url:" + str(start) + " to " + str(start + end))
pass
with open(result_file, "a", encoding="utf-8") as f:
for record in buffer:
f.write(record + "\n")
del buffer[:]
start = end
flag = start
if ((end - old_flag) != 0):
progress = (start - old_flag) / (end - old_flag)
else:
progress = 1
total_time = time.time() - start_time
expected_time_left = total_time / progress - total_time
log_str = "Current Progress: " + str(
progress * 100) + "%, Already run:" + str(
total_time / 60) + "mins, Expected Left:" + str(
expected_time_left / 60) + "mins, flag:" + str(flag)
csv.appendToFile(log_file, log_str)
finish = True
del content[:]
print("Finish Transaction downloading")
def __appendDetailFile(detail_file, txid):
text = str(tq(txid))
CsvIO.appendToFile(detail_file, text)
def classifyTransaction(Trandir, filename_proto=""):
pay_for_fee = []
sent_to = []
receive_from = []
file_list = os.listdir(Trandir)
if (filename_proto == ""):
filename_proto = str(
file_list[0]
)[0:
-5] #hardcode here since the file is produced by this program and will always end with num.csv, [0] will be 1.csv
for file in range(1, len(file_list) + 1):
filename = Trandir + "/" + filename_proto + str(file) + ".csv"
#print(filename)
content = CsvIO.readFile(filename).replace('"', '')
content = content.split("\n")
#content[0] block info
#content[1] title
try:
for i in range(2, len(content) - 1):
info_list = content[i].split(",")
#print(info_list,file,i)
if (info_list[1] != ""):
# receive_from
amount = float(info_list[2])
if amount == 0:
# the website provides information with some mistakes
continue
receive_from.append({
"time": info_list[0],
"from": info_list[1],
"amount": amount,
"balance": float(info_list[5]),
"transaction": info_list[6]
})
elif (info_list[4] != "(fee)"):
# sent_to
amount = float(info_list[3])
if amount == 0:
continue
sent_to.append({
"time": info_list[0],
"to": info_list[4],
"amount": amount,
"balance": float(info_list[5]),
"transaction": info_list[6]
})
else:
#pay_for_fee
pay_for_fee.append({
"time": info_list[0],
"to": info_list[4],
"amount": float(info_list[3]),
"balance": float(info_list[5]),
"transaction": info_list[6]
})
except:
continue
return pay_for_fee, sent_to, receive_from
def beginDownload():
global buffer, thread, error_log, error_url, log_file, store_path, url_file, all_wallet, result_file, content, flag, finish, total_task
pattern2 = re.compile(r'<tr><td><a href=[^>]+')
pattern_balance = re.compile(r'amount">[^&<]+')
pattern_incomTx = re.compile(r"<td>[0-9]+")
pattern_incomBlock = re.compile(r"<td>[0-9]+")
content = csv.readFile(url_file)
content = content.split("\n")
start = flag
old_flag = start
end = len(content)
total_task = end
start_time = time.time()
while start + thread < end:
sub_content = content[start:start + thread]
del buffer[:]
try:
thrs = [
threading.Thread(target=downloadOneUrlPk,
args=[
url, pattern2, pattern_balance,
pattern_incomTx, pattern_incomBlock
]) for url in sub_content
]
[thr.start() for thr in thrs]
[thr.join() for thr in thrs]
except Exception as e:
csv.appendToFile(error_log, str(e))
csv.appendToFile(
error_log,
"Affected url:" + str(start) + " to " + str(start + thread))
pass
with open(result_file, "a", encoding="utf-8") as f:
for record in buffer:
f.write(record + "\n")
start += thread
del buffer[:]
flag = start # update the flag
if ((end - old_flag) != 0):
progress = (start - old_flag) / (end - old_flag)
else:
progress = 1
total_time = time.time() - start_time
expected_time_left = total_time / progress - total_time
log_str = "Current Progress: " + str(
progress * 100) + "%, Already run:" + str(
total_time / 60) + "mins, Expected Left:" + str(
expected_time_left / 60) + "mins" + ",flag:" + str(flag)
csv.appendToFile(log_file, log_str)
sub_content = content[start:end]
del buffer[:]
try:
thrs = [
threading.Thread(target=downloadOneUrlPk,
args=[
url, pattern2, pattern_balance,
pattern_incomTx, pattern_incomBlock
]) for url in sub_content
]
[thr.start() for thr in thrs]
[thr.join() for thr in thrs]
except Exception as e:
csv.appendToFile(error_log, str(e))
csv.appendToFile(
error_log,
"Affected url:" + str(start) + " to " + str(start + end))
pass
with open(result_file, "a", encoding="utf-8") as f:
for record in buffer:
f.write(record + "\n")
del buffer[:]
start = end
flag = start
del buffer[:]
if ((end - old_flag) != 0):
progress = (start - old_flag) / (end - old_flag)
else:
progress = 1
total_time = time.time() - start_time
expected_time_left = total_time / progress - total_time
log_str = "Current Progress: " + str(
progress * 100) + "%, Already run:" + str(
total_time / 60) + "mins, Expected Left:" + str(
expected_time_left / 60) + "mins" + ",flag:" + str(flag)
csv.appendToFile(log_file, log_str)
finish = True
del content[:]
print("Finish wallet public key downloading", total_task, flag)