def extractTxid(wallet_tx_path, txid_store_file):
print("Extracting")
f = open(txid_store_file, 'a', encoding="utf-8")
for filename in os.listdir(wallet_tx_path):
try:
content = csv.readFile(wallet_tx_path + filename)
content = content.split("\n")
all_txid = []
for ii in range(2, len(content) - 1):
all_txid.append(content[ii].split(",")[-1].replace('"', ""))
for id in all_txid:
f.write(id + '\n')
except:
continue
f.close()
print("Deleting duplication")
content = csv.readFile(txid_store_file)
content = content.split("\n")
clear_content = set(content)
try:
clear_content.remove("")
except:
pass
csv.writeToFile(txid_store_file, "") # delete original
f = open(txid_store_file, 'a', encoding="utf-8")
for i in clear_content:
f.write(i + "\n")
f.close()
print("Finish")
def downloadTransactionBetweenTime(wallet_id,
end_time,
start_time,
store_path="",
download_transaction_detail=True,
show_time=False): # [start_time,end_time]
# for easily update, accuracy to date end = 2017-10-26 start = 2017-02-21. The path should be a directory instead of a file
total_page = __getPageNum(wallet_id)
page = range(1, total_page + 1)
#find_end = False
if (store_path == ""):
store_path = wallet_id + "_" + start_time + "To" + end_time
if not os.path.exists(store_path):
os.makedirs(store_path)
if (download_transaction_detail):
append_file = store_path + "/Txdetail"
if not os.path.exists(append_file):
os.makedirs(append_file)
#total_time = 0;
for i in page: #
tx_progress_moniter[wallet_id] = (i - 1) / total_page
detail_file = append_file + "/transactionDetail_" + str(i) + ".json"
#start = time.time()
url = 'https://www.walletexplorer.com/wallet/' + wallet_id + '?page=' + str(
i) + '&format=csv'
local_file = store_path + "/" + wallet_id + "_" + str(i) + '.csv'
data = NetIO.readDataFrom(url)
end, start = __findTime(data)
if (end >= start_time and end <= end_time) or (start >= start_time
and start <= end_time):
CsvIO.writeToFile(local_file, data)
elif (end < start_time):
break
if (download_transaction_detail):
# Also go through the transaction id on every page and append the json to a file
d_data = CsvIO.readFile(local_file)
dd = d_data.split("\n")
all_txid = []
for ii in range(2, len(dd) - 1):
all_txid.append(dd[ii].split(",")[-1].replace('"', ""))
for txid in all_txid:
#bottle neck here
try:
CsvIO.appendToFile(detail_file, str(tq(txid)))
except:
pass
#boost a lot
# thrs = [threading.Thread(target=__appendDetailFile, args=[detail_file,txid]) for txid in all_txid]
# [thr.start() for thr in thrs]
# [thr.join() for thr in thrs]
tx_progress_moniter[wallet_id] = i / total_page
def getHashpub2wallet(hashdir):
the_map = {}
lst = os.listdir(hashdir)
lst.sort()
for file in lst:
content = CsvIO.readFile(hashdir + "/" + file)
content = content.split("\n")
for line in content:
#print(line,file.replace(".csv",""))
the_map[line] = file.replace(
".csv", "") # this filename must be the wallet_id
#print("finish ",file)
return the_map
def downloadTxOnly(txidpath, deduplicate=True, hint=""):
if deduplicate:
content = csv.readFile(txidpath)
content = content.split("\n")
clear_content = set(content)
try:
clear_content.remove("")
except:
pass
csv.writeToFile(txidpath, "") # delete original
f = open(txidpath, 'a', encoding="utf-8")
for i in clear_content:
f.write(i + "\n")
f.close()
DetailedTransactionDownloader.init()
DetailedTransactionDownloader.url_file = txidpath
main_thred = threading.Thread(target=DetailedTransactionDownloader.main)
moniter = threading.Thread(target=addMonitorFor,
args=[DetailedTransactionDownloader, 30])
main_thred.start()
moniter.start()
main_thred.join()
moniter.join()
def beginDownload():
global buffer, thread, error_log, error_url, log_file, store_path, url_file, result_file, flag, content, finish, total_task
content = csv.readFile(url_file)
content = content.split("\n")
start = flag
old_flag = start
end = len(content)
total_task = end
start_time = time.time()
while start + thread < end:
sub_content = content[start:start + thread]
del buffer[:]
try:
thrs = [
threading.Thread(target=downloadOneTx, args=[url])
for url in sub_content
]
[thr.start() for thr in thrs]
[thr.join() for thr in thrs]
except Exception as e:
csv.appendToFile(error_log, str(e))
csv.appendToFile(
error_log,
"Affected url:" + str(start) + " to " + str(start + thread))
pass
with open(result_file, "a", encoding="utf-8") as f:
for record in buffer:
f.write(record + "\n")
del buffer[:]
start += thread
flag = start
if ((end - old_flag) != 0):
progress = (start - old_flag) / (end - old_flag)
else:
progress = 1
total_time = time.time() - start_time
expected_time_left = total_time / progress - total_time
log_str = "Current Progress: " + str(
progress * 100) + "%, Already run:" + str(
total_time / 60) + "mins, Expected Left:" + str(
expected_time_left / 60) + "mins, flag:" + str(flag)
csv.appendToFile(log_file, log_str)
sub_content = content[start:end]
del buffer[:]
try:
thrs = [
threading.Thread(target=downloadOneTx, args=[url])
for url in sub_content
]
[thr.start() for thr in thrs]
[thr.join() for thr in thrs]
except Exception as e:
csv.appendToFile(error_log, str(e))
csv.appendToFile(
error_log,
"Affected url:" + str(start) + " to " + str(start + end))
pass
with open(result_file, "a", encoding="utf-8") as f:
for record in buffer:
f.write(record + "\n")
del buffer[:]
start = end
flag = start
if ((end - old_flag) != 0):
progress = (start - old_flag) / (end - old_flag)
else:
progress = 1
total_time = time.time() - start_time
expected_time_left = total_time / progress - total_time
log_str = "Current Progress: " + str(
progress * 100) + "%, Already run:" + str(
total_time / 60) + "mins, Expected Left:" + str(
expected_time_left / 60) + "mins, flag:" + str(flag)
csv.appendToFile(log_file, log_str)
finish = True
del content[:]
print("Finish Transaction downloading")
def classifyTransaction(Trandir, filename_proto=""):
pay_for_fee = []
sent_to = []
receive_from = []
file_list = os.listdir(Trandir)
if (filename_proto == ""):
filename_proto = str(
file_list[0]
)[0:
-5] #hardcode here since the file is produced by this program and will always end with num.csv, [0] will be 1.csv
for file in range(1, len(file_list) + 1):
filename = Trandir + "/" + filename_proto + str(file) + ".csv"
#print(filename)
content = CsvIO.readFile(filename).replace('"', '')
content = content.split("\n")
#content[0] block info
#content[1] title
try:
for i in range(2, len(content) - 1):
info_list = content[i].split(",")
#print(info_list,file,i)
if (info_list[1] != ""):
# receive_from
amount = float(info_list[2])
if amount == 0:
# the website provides information with some mistakes
continue
receive_from.append({
"time": info_list[0],
"from": info_list[1],
"amount": amount,
"balance": float(info_list[5]),
"transaction": info_list[6]
})
elif (info_list[4] != "(fee)"):
# sent_to
amount = float(info_list[3])
if amount == 0:
continue
sent_to.append({
"time": info_list[0],
"to": info_list[4],
"amount": amount,
"balance": float(info_list[5]),
"transaction": info_list[6]
})
else:
#pay_for_fee
pay_for_fee.append({
"time": info_list[0],
"to": info_list[4],
"amount": float(info_list[3]),
"balance": float(info_list[5]),
"transaction": info_list[6]
})
except:
continue
return pay_for_fee, sent_to, receive_from
def beginDownload():
global buffer, thread, error_log, error_url, log_file, store_path, url_file, all_wallet, result_file, content, flag, finish, total_task
pattern2 = re.compile(r'<tr><td><a href=[^>]+')
pattern_balance = re.compile(r'amount">[^&<]+')
pattern_incomTx = re.compile(r"<td>[0-9]+")
pattern_incomBlock = re.compile(r"<td>[0-9]+")
content = csv.readFile(url_file)
content = content.split("\n")
start = flag
old_flag = start
end = len(content)
total_task = end
start_time = time.time()
while start + thread < end:
sub_content = content[start:start + thread]
del buffer[:]
try:
thrs = [
threading.Thread(target=downloadOneUrlPk,
args=[
url, pattern2, pattern_balance,
pattern_incomTx, pattern_incomBlock
]) for url in sub_content
]
[thr.start() for thr in thrs]
[thr.join() for thr in thrs]
except Exception as e:
csv.appendToFile(error_log, str(e))
csv.appendToFile(
error_log,
"Affected url:" + str(start) + " to " + str(start + thread))
pass
with open(result_file, "a", encoding="utf-8") as f:
for record in buffer:
f.write(record + "\n")
start += thread
del buffer[:]
flag = start # update the flag
if ((end - old_flag) != 0):
progress = (start - old_flag) / (end - old_flag)
else:
progress = 1
total_time = time.time() - start_time
expected_time_left = total_time / progress - total_time
log_str = "Current Progress: " + str(
progress * 100) + "%, Already run:" + str(
total_time / 60) + "mins, Expected Left:" + str(
expected_time_left / 60) + "mins" + ",flag:" + str(flag)
csv.appendToFile(log_file, log_str)
sub_content = content[start:end]
del buffer[:]
try:
thrs = [
threading.Thread(target=downloadOneUrlPk,
args=[
url, pattern2, pattern_balance,
pattern_incomTx, pattern_incomBlock
]) for url in sub_content
]
[thr.start() for thr in thrs]
[thr.join() for thr in thrs]
except Exception as e:
csv.appendToFile(error_log, str(e))
csv.appendToFile(
error_log,
"Affected url:" + str(start) + " to " + str(start + end))
pass
with open(result_file, "a", encoding="utf-8") as f:
for record in buffer:
f.write(record + "\n")
del buffer[:]
start = end
flag = start
del buffer[:]
if ((end - old_flag) != 0):
progress = (start - old_flag) / (end - old_flag)
else:
progress = 1
total_time = time.time() - start_time
expected_time_left = total_time / progress - total_time
log_str = "Current Progress: " + str(
progress * 100) + "%, Already run:" + str(
total_time / 60) + "mins, Expected Left:" + str(
expected_time_left / 60) + "mins" + ",flag:" + str(flag)
csv.appendToFile(log_file, log_str)
finish = True
del content[:]
print("Finish wallet public key downloading", total_task, flag)