def map_coordinates_parallel(input, coordinates, output=None, order=3, mode='constant', cval=0.0,
prefilter=True, chunklen=None, threads=None):
"""
Parallalized version of `scipy.ndimage.map_coordinates`.
`scipy.ndimage.map_coordinates` is slow for large datasets. Speed improvement can be
achieved by
* Splitting the data into chunks
* Performing the transformation of chunks in parallel
New parameters:
chunklen: Size of the chunks in pixels per axis. Default: None
Special values:
None: Automatic (Chooses a default based on number of dimensions)
0: Do not split data into chunks. (implicitly sets threads==1)
threads: Number of threads. Default: None
None: Automatic (One thread per available processing unit)
"""
# this part is taken without change from scipy's serial implementation
if order < 0 or order > 5:
raise RuntimeError('spline order not supported')
input = np.asarray(input)
if np.iscomplexobj(input):
raise TypeError('Complex type not supported')
coordinates = np.asarray(coordinates)
if np.iscomplexobj(coordinates):
raise TypeError('Complex type not supported')
output_shape = coordinates.shape[1:]
if input.ndim < 1 or len(output_shape) < 1:
raise RuntimeError('input and output rank must be > 0')
if coordinates.shape[0] != input.ndim:
raise RuntimeError('invalid shape for coordinate array')
mode = _ni_support._extend_mode_to_code(mode)
if prefilter and order > 1:
filtered = spline_filter(input, order, output=np.float64)
else:
filtered = input
# return value of `_ni_support._get_output` changed between scipy versions, code here is
# adapted to work with both
output = _ni_support._get_output(output, input, shape=output_shape)
retval = output
if isinstance(output, tuple):
output, retval = output
# below here there is the new code for splitting into chunks and parallel execution
if chunklen is None:
# set defaults
chunklen = 128
if output.ndim < 3:
chunklen = 1024
def chunk_arguments(filtered, coordinates, output):
chunks = []
for axis in range(output.ndim):
chunkstarts = np.arange(0, output.shape[axis], chunklen)
chunkends = chunkstarts + chunklen
chunkends[-1] = output.shape[axis]
chunks.append([slice(start, stop) for start, stop in zip(chunkstarts, chunkends)])
for chunk in itertools.product(*chunks):
sub_coordinates = coordinates[(slice(None),) + chunk].copy()
filtered_region = []
for in_axis in range(filtered.ndim):
c = sub_coordinates[in_axis, ...]
cmin = max(0, int(np.floor(np.min(c)))-5)
cmax = min(filtered.shape[in_axis], int(np.ceil(np.max(c)))+5)
sub_coordinates[in_axis, ...] -= cmin
filtered_region.append(slice(cmin, cmax))
sub_filtered = filtered[tuple(filtered_region)]
sub_output = output[chunk]
yield (sub_filtered, sub_coordinates, sub_output)
def map_coordinates_chunk(arg):
sub_filtered, sub_coordinates, sub_output = arg
_nd_image.geometric_transform(sub_filtered, None, sub_coordinates, None, None,
sub_output, order, mode, cval, None, None)
if chunklen > 0:
list_of_chunk_args = list(chunk_arguments(filtered, coordinates, output))
else:
list_of_chunk_args = [(filtered, coordinates, output)]
if len(list_of_chunk_args) == 1:
threads = 1
if threads != 1:
threadpool = ThreadPoolExecutor(threads)
my_map = threadpool.map
else:
my_map = map
# execution happens here
list(my_map(map_coordinates_chunk, list_of_chunk_args))
if threads != 1:
if have_concurrent_futures:
threadpool.shutdown()
else:
threadpool.close()
threadpool.join()
return retval
def blocks(self,
start=None,
stop=None,
max_batch_size=None,
threading=False,
thread_num=8,
only_ops=False,
only_virtual_ops=False):
""" Yields blocks starting from ``start``.
:param int start: Starting block
:param int stop: Stop at this block
:param int max_batch_size: only for appbase nodes. When not None, batch calls of are used.
Cannot be combined with threading
:param bool threading: Enables threading. Cannot be combined with batch calls
:param int thread_num: Defines the number of threads, when `threading` is set.
:param bool only_ops: Only yield operations (default: False).
Cannot be combined with ``only_virtual_ops=True``.
:param bool only_virtual_ops: Only yield virtual operations (default: False)
.. note:: If you want instant confirmation, you need to instantiate
class:`beem.blockchain.Blockchain` with
``mode="head"``, otherwise, the call will wait until
confirmed in an irreversible block.
"""
# Let's find out how often blocks are generated!
current_block = self.get_current_block()
current_block_num = current_block.block_num
if not start:
start = current_block_num
head_block_reached = False
if threading and FUTURES_MODULE is not None:
pool = ThreadPoolExecutor(max_workers=thread_num)
elif threading:
pool = Pool(thread_num, batch_mode=True)
if threading:
steem_instance = [self.steem]
nodelist = self.steem.rpc.nodes.export_working_nodes()
for i in range(thread_num - 1):
steem_instance.append(
stm.Steem(node=nodelist,
num_retries=self.steem.rpc.num_retries,
num_retries_call=self.steem.rpc.num_retries_call,
timeout=self.steem.rpc.timeout))
# We are going to loop indefinitely
latest_block = 0
while True:
if stop:
head_block = stop
else:
current_block_num = self.get_current_block_num()
head_block = current_block_num
if threading and not head_block_reached:
latest_block = start - 1
result_block_nums = []
for blocknum in range(start, head_block + 1, thread_num):
# futures = []
i = 0
if FUTURES_MODULE is not None:
futures = []
block_num_list = []
# freeze = self.steem.rpc.nodes.freeze_current_node
num_retries = self.steem.rpc.nodes.num_retries
# self.steem.rpc.nodes.freeze_current_node = True
self.steem.rpc.nodes.num_retries = thread_num
error_cnt = self.steem.rpc.nodes.node.error_cnt
while i < thread_num and blocknum + i <= head_block:
block_num_list.append(blocknum + i)
results = []
if FUTURES_MODULE is not None:
futures.append(
pool.submit(Block,
blocknum + i,
only_ops=only_ops,
only_virtual_ops=only_virtual_ops,
steem_instance=steem_instance[i]))
else:
pool.enqueue(Block,
blocknum + i,
only_ops=only_ops,
only_virtual_ops=only_virtual_ops,
steem_instance=steem_instance[i])
i += 1
if FUTURES_MODULE is not None:
try:
results = [
r.result() for r in as_completed(futures)
]
except Exception as e:
log.error(str(e))
else:
pool.run(True)
pool.join()
for result in pool.results():
results.append(result)
pool.abort()
self.steem.rpc.nodes.num_retries = num_retries
# self.steem.rpc.nodes.freeze_current_node = freeze
new_error_cnt = self.steem.rpc.nodes.node.error_cnt
self.steem.rpc.nodes.node.error_cnt = error_cnt
if new_error_cnt > error_cnt:
self.steem.rpc.nodes.node.error_cnt += 1
# self.steem.rpc.next()
checked_results = []
for b in results:
if b.block_num is not None and int(
b.block_num) not in result_block_nums:
b["id"] = b.block_num
b.identifier = b.block_num
checked_results.append(b)
result_block_nums.append(int(b.block_num))
missing_block_num = list(
set(block_num_list).difference(set(result_block_nums)))
while len(missing_block_num) > 0:
for blocknum in missing_block_num:
try:
block = Block(
blocknum,
only_ops=only_ops,
only_virtual_ops=only_virtual_ops,
steem_instance=self.steem)
checked_results.append(block)
result_block_nums.append(int(block.block_num))
except Exception as e:
log.error(str(e))
missing_block_num = list(
set(block_num_list).difference(
set(result_block_nums)))
from operator import itemgetter
blocks = sorted(checked_results, key=itemgetter('id'))
for b in blocks:
if latest_block < int(b.block_num):
latest_block = int(b.block_num)
yield b
if latest_block <= head_block:
for blocknum in range(latest_block + 1, head_block + 1):
if blocknum not in result_block_nums:
block = Block(blocknum,
only_ops=only_ops,
only_virtual_ops=only_virtual_ops,
steem_instance=self.steem)
result_block_nums.append(blocknum)
yield block
elif max_batch_size is not None and (
head_block -
start) >= max_batch_size and not head_block_reached:
if not self.steem.is_connected():
raise OfflineHasNoRPCException(
"No RPC available in offline mode!")
self.steem.rpc.set_next_node_on_empty_reply(False)
latest_block = start - 1
batches = max_batch_size
for blocknumblock in range(start, head_block + 1, batches):
# Get full block
if (head_block - blocknumblock) < batches:
batches = head_block - blocknumblock + 1
for blocknum in range(blocknumblock,
blocknumblock + batches - 1):
if only_virtual_ops:
if self.steem.rpc.get_use_appbase():
# self.steem.rpc.get_ops_in_block({"block_num": blocknum, 'only_virtual': only_virtual_ops}, api="account_history", add_to_queue=True)
self.steem.rpc.get_ops_in_block(
blocknum,
only_virtual_ops,
add_to_queue=True)
else:
self.steem.rpc.get_ops_in_block(
blocknum,
only_virtual_ops,
add_to_queue=True)
else:
if self.steem.rpc.get_use_appbase():
self.steem.rpc.get_block(
{"block_num": blocknum},
api="block",
add_to_queue=True)
else:
self.steem.rpc.get_block(blocknum,
add_to_queue=True)
latest_block = blocknum
if batches >= 1:
latest_block += 1
if latest_block <= head_block:
if only_virtual_ops:
if self.steem.rpc.get_use_appbase():
# self.steem.rpc.get_ops_in_block({"block_num": blocknum, 'only_virtual': only_virtual_ops}, api="account_history", add_to_queue=False)
block_batch = self.steem.rpc.get_ops_in_block(
blocknum,
only_virtual_ops,
add_to_queue=False)
else:
block_batch = self.steem.rpc.get_ops_in_block(
blocknum,
only_virtual_ops,
add_to_queue=False)
else:
if self.steem.rpc.get_use_appbase():
block_batch = self.steem.rpc.get_block(
{"block_num": latest_block},
api="block",
add_to_queue=False)
else:
block_batch = self.steem.rpc.get_block(
latest_block, add_to_queue=False)
if not bool(block_batch):
raise BatchedCallsNotSupported()
blocknum = latest_block - len(block_batch) + 1
if not isinstance(block_batch, list):
block_batch = [block_batch]
for block in block_batch:
if not bool(block):
continue
if self.steem.rpc.get_use_appbase():
if only_virtual_ops:
block = block["ops"]
else:
block = block["block"]
block = Block(block,
only_ops=only_ops,
only_virtual_ops=only_virtual_ops,
steem_instance=self.steem)
block["id"] = block.block_num
block.identifier = block.block_num
yield block
blocknum = block.block_num
else:
# Blocks from start until head block
for blocknum in range(start, head_block + 1):
# Get full block
block = self.wait_for_and_get_block(
blocknum,
only_ops=only_ops,
only_virtual_ops=only_virtual_ops,
block_number_check_cnt=5,
last_current_block_num=current_block_num)
yield block
# Set new start
start = head_block + 1
head_block_reached = True
if stop and start > stop:
return
# Sleep for one block
time.sleep(self.block_interval)
def map_coordinates_parallel(input,
coordinates,
output=None,
order=3,
mode='constant',
cval=0.0,
prefilter=True,
chunklen=None,
threads=None):
"""
Parallalized version of `scipy.ndimage.map_coordinates`.
`scipy.ndimage.map_coordinates` is slow for large datasets. Speed improvement can be
achieved by
* Splitting the data into chunks
* Performing the transformation of chunks in parallel
New parameters:
chunklen: Size of the chunks in pixels per axis. Default: None
Special values:
None: Automatic (Chooses a default based on number of dimensions)
0: Do not split data into chunks. (implicitly sets threads==1)
threads: Number of threads. Default: None
None: Automatic (One thread per available processing unit)
"""
# this part is taken without change from scipy's serial implementation
if order < 0 or order > 5:
raise RuntimeError('spline order not supported')
input = np.asarray(input)
if np.iscomplexobj(input):
raise TypeError('Complex type not supported')
coordinates = np.asarray(coordinates)
if np.iscomplexobj(coordinates):
raise TypeError('Complex type not supported')
output_shape = coordinates.shape[1:]
if input.ndim < 1 or len(output_shape) < 1:
raise RuntimeError('input and output rank must be > 0')
if coordinates.shape[0] != input.ndim:
raise RuntimeError('invalid shape for coordinate array')
mode = _ni_support._extend_mode_to_code(mode)
if prefilter and order > 1:
filtered = spline_filter(input, order, output=np.float64)
else:
filtered = input
# return value of `_ni_support._get_output` changed between scipy versions, code here is
# adapted to work with both
output = _ni_support._get_output(output, input, shape=output_shape)
retval = output
if isinstance(output, tuple):
output, retval = output
# below here there is the new code for splitting into chunks and parallel execution
if chunklen is None:
# set defaults
chunklen = 128
if output.ndim < 3:
chunklen = 1024
def chunk_arguments(filtered, coordinates, output):
chunks = []
for axis in range(output.ndim):
chunkstarts = np.arange(0, output.shape[axis], chunklen)
chunkends = chunkstarts + chunklen
chunkends[-1] = output.shape[axis]
chunks.append([
slice(start, stop)
for start, stop in zip(chunkstarts, chunkends)
])
for chunk in itertools.product(*chunks):
sub_coordinates = coordinates[(slice(None), ) + chunk].copy()
filtered_region = []
for in_axis in range(filtered.ndim):
c = sub_coordinates[in_axis, ...]
cmin = max(0, int(np.floor(np.min(c))) - 5)
cmax = min(filtered.shape[in_axis],
int(np.ceil(np.max(c))) + 5)
sub_coordinates[in_axis, ...] -= cmin
filtered_region.append(slice(cmin, cmax))
sub_filtered = filtered[tuple(filtered_region)]
sub_output = output[chunk]
yield (sub_filtered, sub_coordinates, sub_output)
def map_coordinates_chunk(arg):
sub_filtered, sub_coordinates, sub_output = arg
_nd_image.geometric_transform(sub_filtered, None, sub_coordinates,
None, None, sub_output, order, mode,
cval, None, None)
if chunklen > 0:
list_of_chunk_args = list(
chunk_arguments(filtered, coordinates, output))
else:
list_of_chunk_args = [(filtered, coordinates, output)]
if len(list_of_chunk_args) == 1:
threads = 1
if threads != 1:
threadpool = ThreadPoolExecutor(threads)
my_map = threadpool.map
else:
my_map = map
# execution happens here
list(my_map(map_coordinates_chunk, list_of_chunk_args))
if threads != 1:
if have_concurrent_futures:
threadpool.shutdown()
else:
threadpool.close()
threadpool.join()
return retval
while True:
os.system("clear")
time.sleep(10)
if __name__ == '__main__':
lock = threading.Lock() #初始化一把锁
pool = ThreadPoolExecutor(max_workers=5)
try:
with open("./config.yaml", "r",encoding='utf-8') as yaml_file:
yaml_obj = yaml.load(yaml_file.read())
USERNAME = yaml_obj['USERNAME']
PASSWORD = yaml_obj['PASSWORD']
for platform in list(yaml_obj.keys())[2:]:
for massage in yaml_obj[platform]:
achor_name = massage
upload_platform = ''
if len(yaml_obj[platform][massage])==2:
rid = yaml_obj[platform][massage][0]
upload_platform = yaml_obj[platform][massage][1]
else:rid = yaml_obj[platform][massage][0]
pool.submit(download, achor_name,rid,platform,upload_platform,USERNAME,PASSWORD,lock)
pool.submit(clearshell)
pool.close()
pool.join()
except Exception as err:
print(err)
time.sleep(10)
else:
path += '.png'
download(path, url)
return
def download(path, url):
try:
photodata = requests.get(url).content
with open(r'bqb\%s' % path, 'wb') as w:
w.write(photodata)
print(path + '下载完成')
return
except:
print(path + "下载错误！")
return
if __name__ == "__main__":
start = time.time()
urllist = [
'https://www.fabiaoqing.com/biaoqing/lists/page/%d.html' % i
for i in range(1, 5)
]
pool = ThreadPoolExecutor(4)
while len(urllist):
pool.submit(Data, urllist[0])
pool.join(2)
urllist.pop(0)
print("耗时:", time.time() - start)
