def downloadAllPublicAddressOf(wallet_id,
local_file="",
start_page=1,
show_time=False):
if local_file == "":
local_file = wallet_id + ".csv" # if the user doesn't provide a file to store the key, the function will generate one
total_page = __getKeyPageNum(wallet_id)
total_time = 0
for i in range(start_page, total_page + 1):
start = time.time()
url = 'https://www.walletexplorer.com/wallet/' + wallet_id + '/addresses?page=' + str(
i)
data = NetIO.readDataFrom(url)
flag = 0
pattern2 = re.compile(r'<tr><td><a href=[^>]+')
match = True
while match:
match = pattern2.search(data, flag)
if match:
flag = int(match.span()[-1])
sub = match.group()[26:-1]
CsvIO.appendToFile(local_file, sub)
else:
break
finish = time.time()
t = finish - start
total_time += t
expect_left = (total_page - i) * t
if show_time:
print(str(i), 'tooks ', t, "secs")
print(total_page, "time left:", expect_left / 60, "mins")
def downloadAllPublicAddressOf(wallet_id,
local_file="",
start_page=1,
show_time=False,
clear_file=False):
if local_file == "":
local_file = wallet_id + ".csv" # if the user doesn't provide a file to store the key, the function will generate one
if (clear_file):
CsvIO.writeToFile(local_file, "") #Clear the file first
total_page = __getKeyPageNum(wallet_id)
#total_time = 0;
pattern2 = re.compile(r'<tr><td><a href=[^>]+')
pattern_balance = re.compile(r'amount">[^&<]+')
pattern_incomTx = re.compile(r"<td>[0-9]+")
pattern_incomBlock = re.compile(r"<td>[0-9]+")
for i in range(start_page, total_page + 1):
pk_progress_moniter[wallet_id] = (i - 1) / total_page
#start = time.time()
url = 'https://www.walletexplorer.com/wallet/' + wallet_id + '/addresses?page=' + str(
i)
data = NetIO.readDataFrom(url)
flag = 0
match = True
while match:
match = pattern2.search(data, flag)
if match:
flag = int(match.span()[-1])
sub = match.group()[26:-1]
match_balance = pattern_balance.search(data, flag)
flag = int(match_balance.span()[-1])
balance = match_balance.group()[8:]
match_incomTx = pattern_incomTx.search(data, flag)
flag = int(match_incomTx.span()[-1])
incomTx = match_incomTx.group()[4:]
match_block = pattern_incomBlock.search(data, flag)
flag = int(match_block.span()[-1])
blockID = match_block.group()[4:]
#print(sub+","+balance+","+incomTx+","+blockID)
CsvIO.appendToFile(
local_file,
sub + "," + balance + "," + incomTx + "," + blockID)
else:
break
pk_progress_moniter[wallet_id] = (i) / total_page
def downloadTransactionBetweenTime(wallet_id,
end_time,
start_time,
store_path="",
download_transaction_detail=True,
show_time=False): # [start_time,end_time]
# for easily update, accuracy to date end = 2017-10-26 start = 2017-02-21. The path should be a directory instead of a file
total_page = __getPageNum(wallet_id)
page = range(1, total_page + 1)
#find_end = False
if (store_path == ""):
store_path = wallet_id + "_" + start_time + "To" + end_time
if not os.path.exists(store_path):
os.makedirs(store_path)
if (download_transaction_detail):
append_file = store_path + "/Txdetail"
if not os.path.exists(append_file):
os.makedirs(append_file)
#total_time = 0;
for i in page: #
tx_progress_moniter[wallet_id] = (i - 1) / total_page
detail_file = append_file + "/transactionDetail_" + str(i) + ".json"
#start = time.time()
url = 'https://www.walletexplorer.com/wallet/' + wallet_id + '?page=' + str(
i) + '&format=csv'
local_file = store_path + "/" + wallet_id + "_" + str(i) + '.csv'
data = NetIO.readDataFrom(url)
end, start = __findTime(data)
if (end >= start_time and end <= end_time) or (start >= start_time
and start <= end_time):
CsvIO.writeToFile(local_file, data)
elif (end < start_time):
break
if (download_transaction_detail):
# Also go through the transaction id on every page and append the json to a file
d_data = CsvIO.readFile(local_file)
dd = d_data.split("\n")
all_txid = []
for ii in range(2, len(dd) - 1):
all_txid.append(dd[ii].split(",")[-1].replace('"', ""))
for txid in all_txid:
#bottle neck here
try:
CsvIO.appendToFile(detail_file, str(tq(txid)))
except:
pass
#boost a lot
# thrs = [threading.Thread(target=__appendDetailFile, args=[detail_file,txid]) for txid in all_txid]
# [thr.start() for thr in thrs]
# [thr.join() for thr in thrs]
tx_progress_moniter[wallet_id] = i / total_page
def downloadOneTx(url):
global buffer, thread, error_log, error_url, log_file, store_path, url_file, error_num, result_file
if (url == ""):
return
try:
result = str(getBlockInfoTran(url))
buffer.append(result)
except Exception as e:
csv.appendToFile(error_log, str(e))
csv.appendToFile(error_url, url)
error_num += 1
def addMonitorFor(target_function, time_inter):
while (not target_function.finish):
remember = target_function.flag
time.sleep(time_inter)
if (remember == target_function.flag):
csv.appendToFile(
center_log,
str(time.time()) + ":" + str(target_function) +
" stopped, restoring...")
main_thred = threading.Thread(target=target_function.restore)
moniter = threading.Thread(target=addMonitorFor,
args=[target_function, time_inter])
main_thred.start()
moniter.start()
main_thred.join()
moniter.join()
def downloadOneUrlPk(url, pattern2, pattern_balance, pattern_incomTx,
pattern_incomBlock):
global buffer, thread, error_log, error_url, log_file, store_path, url_file, all_wallet, result_file
if (url == ""):
return
try:
data = NetIO.readDataFrom(url)
local_flag = 0
match = True
while match:
match = pattern2.search(data, local_flag)
if match:
local_flag = int(match.span()[-1])
sub = match.group()[26:-1]
match_balance = pattern_balance.search(data, local_flag)
local_flag = int(match_balance.span()[-1])
balance = match_balance.group()[8:]
match_incomTx = pattern_incomTx.search(data, local_flag)
local_flag = int(match_incomTx.span()[-1])
incomTx = match_incomTx.group()[4:]
match_block = pattern_incomBlock.search(data, local_flag)
local_flag = int(match_block.span()[-1])
blockID = match_block.group()[4:]
# print(sub+","+balance+","+incomTx+","+blockID)
buffer.append(
util.find_between(
url, "https://www.walletexplorer.com/wallet/",
"/addresses") + "," + sub + "," + balance + "," +
incomTx + "," + blockID)
else:
break
except Exception as e:
csv.appendToFile(error_log, str(e))
csv.appendToFile(error_url, url)
#Call main to start
#main()
def downloadOneCSV(url):
global buffer, thread, error_log, error_url, log_file, store_path, url_file, all_wallet
if (url == ""):
return
try:
filename = url.replace("https://www.walletexplorer.com/wallet/",
"").replace("?",
"_").replace("format=csv", ".csv")
content = NetIO.readDataFrom(url)
csv.writeToFile(store_path + filename, content)
except Exception as e:
csv.appendToFile(error_log, str(e))
csv.appendToFile(error_url, url)
#Call main to start
#main()
def beginDownload():
global buffer, thread, error_log, error_url, log_file, store_path, url_file, result_file, flag, content, finish, total_task
content = csv.readFile(url_file)
content = content.split("\n")
start = flag
old_flag = start
end = len(content)
total_task = end
start_time = time.time()
while start + thread < end:
sub_content = content[start:start + thread]
del buffer[:]
try:
thrs = [
threading.Thread(target=downloadOneTx, args=[url])
for url in sub_content
]
[thr.start() for thr in thrs]
[thr.join() for thr in thrs]
except Exception as e:
csv.appendToFile(error_log, str(e))
csv.appendToFile(
error_log,
"Affected url:" + str(start) + " to " + str(start + thread))
pass
with open(result_file, "a", encoding="utf-8") as f:
for record in buffer:
f.write(record + "\n")
del buffer[:]
start += thread
flag = start
if ((end - old_flag) != 0):
progress = (start - old_flag) / (end - old_flag)
else:
progress = 1
total_time = time.time() - start_time
expected_time_left = total_time / progress - total_time
log_str = "Current Progress: " + str(
progress * 100) + "%, Already run:" + str(
total_time / 60) + "mins, Expected Left:" + str(
expected_time_left / 60) + "mins, flag:" + str(flag)
csv.appendToFile(log_file, log_str)
sub_content = content[start:end]
del buffer[:]
try:
thrs = [
threading.Thread(target=downloadOneTx, args=[url])
for url in sub_content
]
[thr.start() for thr in thrs]
[thr.join() for thr in thrs]
except Exception as e:
csv.appendToFile(error_log, str(e))
csv.appendToFile(
error_log,
"Affected url:" + str(start) + " to " + str(start + end))
pass
with open(result_file, "a", encoding="utf-8") as f:
for record in buffer:
f.write(record + "\n")
del buffer[:]
start = end
flag = start
if ((end - old_flag) != 0):
progress = (start - old_flag) / (end - old_flag)
else:
progress = 1
total_time = time.time() - start_time
expected_time_left = total_time / progress - total_time
log_str = "Current Progress: " + str(
progress * 100) + "%, Already run:" + str(
total_time / 60) + "mins, Expected Left:" + str(
expected_time_left / 60) + "mins, flag:" + str(flag)
csv.appendToFile(log_file, log_str)
finish = True
del content[:]
print("Finish Transaction downloading")
def __appendDetailFile(detail_file, txid):
text = str(tq(txid))
CsvIO.appendToFile(detail_file, text)
def beginDownload():
global buffer, thread, error_log, error_url, log_file, store_path, url_file, all_wallet, result_file, content, flag, finish, total_task
pattern2 = re.compile(r'<tr><td><a href=[^>]+')
pattern_balance = re.compile(r'amount">[^&<]+')
pattern_incomTx = re.compile(r"<td>[0-9]+")
pattern_incomBlock = re.compile(r"<td>[0-9]+")
content = csv.readFile(url_file)
content = content.split("\n")
start = flag
old_flag = start
end = len(content)
total_task = end
start_time = time.time()
while start + thread < end:
sub_content = content[start:start + thread]
del buffer[:]
try:
thrs = [
threading.Thread(target=downloadOneUrlPk,
args=[
url, pattern2, pattern_balance,
pattern_incomTx, pattern_incomBlock
]) for url in sub_content
]
[thr.start() for thr in thrs]
[thr.join() for thr in thrs]
except Exception as e:
csv.appendToFile(error_log, str(e))
csv.appendToFile(
error_log,
"Affected url:" + str(start) + " to " + str(start + thread))
pass
with open(result_file, "a", encoding="utf-8") as f:
for record in buffer:
f.write(record + "\n")
start += thread
del buffer[:]
flag = start # update the flag
if ((end - old_flag) != 0):
progress = (start - old_flag) / (end - old_flag)
else:
progress = 1
total_time = time.time() - start_time
expected_time_left = total_time / progress - total_time
log_str = "Current Progress: " + str(
progress * 100) + "%, Already run:" + str(
total_time / 60) + "mins, Expected Left:" + str(
expected_time_left / 60) + "mins" + ",flag:" + str(flag)
csv.appendToFile(log_file, log_str)
sub_content = content[start:end]
del buffer[:]
try:
thrs = [
threading.Thread(target=downloadOneUrlPk,
args=[
url, pattern2, pattern_balance,
pattern_incomTx, pattern_incomBlock
]) for url in sub_content
]
[thr.start() for thr in thrs]
[thr.join() for thr in thrs]
except Exception as e:
csv.appendToFile(error_log, str(e))
csv.appendToFile(
error_log,
"Affected url:" + str(start) + " to " + str(start + end))
pass
with open(result_file, "a", encoding="utf-8") as f:
for record in buffer:
f.write(record + "\n")
del buffer[:]
start = end
flag = start
del buffer[:]
if ((end - old_flag) != 0):
progress = (start - old_flag) / (end - old_flag)
else:
progress = 1
total_time = time.time() - start_time
expected_time_left = total_time / progress - total_time
log_str = "Current Progress: " + str(
progress * 100) + "%, Already run:" + str(
total_time / 60) + "mins, Expected Left:" + str(
expected_time_left / 60) + "mins" + ",flag:" + str(flag)
csv.appendToFile(log_file, log_str)
finish = True
del content[:]
print("Finish wallet public key downloading", total_task, flag)