def progress(monitor):
'''
Uses wget's bar_adaptive to remain in conformance with get_file().
:param monitor: And instance of MultipartEncoderMonitor
'''
return wget.callback_progress(monitor.bytes_read, 1, monitor.len,
wget.bar_adaptive)
def callback_charged(blocks, block_size, total_size):
# 'closure' to set bar drawing function in callback
callback_progress(blocks,
block_size,
total_size,
bar_function=bar_function)
def callback_charged(blocks, block_size, total_size):
# 'closure' to set bar drawing function in callback
callback_progress(blocks, block_size, total_size,
bar_function=bar_function)
"values(%s, %s, %s, %s, %s, %s)")
time.sleep(1)
if limit_select != 0:
allRows = allRows[0:limit_select]
tot = len(allRows)
addresses_group = {}
tested_prefixes = []
n = 0
i = 0
for (rank, url) in allRows:
try:
i += 1
wget.callback_progress(i, 1, tot, wget.bar_adaptive)
address = get_ip(url)
addressPrefix = pref(address)
if addressPrefix not in tested_prefixes:
tested_prefixes.append(addressPrefix)
if verbose:
print url, address, addressPrefix, rank
print "Adding work on ", address, ". Number of active threads :", threading.active_count(
)
addresses_group[
address] = url # Send this to all consumer threads ;-)
n += 1
if n >= addr_parallel:
work_queue.put(addresses_group)
addresses_group = {}
n = 0
def progress(monitor):
return wget.callback_progress(monitor.bytes_read, 1,
monitor.encoder.len, wget.bar_adaptive)