while True: os.system("clear") time.sleep(10) if __name__ == '__main__': lock = threading.Lock() #初始化一把锁 pool = ThreadPoolExecutor(max_workers=5) try: with open("./config.yaml", "r",encoding='utf-8') as yaml_file: yaml_obj = yaml.load(yaml_file.read()) USERNAME = yaml_obj['USERNAME'] PASSWORD = yaml_obj['PASSWORD'] for platform in list(yaml_obj.keys())[2:]: for massage in yaml_obj[platform]: achor_name = massage upload_platform = '' if len(yaml_obj[platform][massage])==2: rid = yaml_obj[platform][massage][0] upload_platform = yaml_obj[platform][massage][1] else:rid = yaml_obj[platform][massage][0] pool.submit(download, achor_name,rid,platform,upload_platform,USERNAME,PASSWORD,lock) pool.submit(clearshell) pool.close() pool.join() except Exception as err: print(err) time.sleep(10)
def map_coordinates_parallel(input, coordinates, output=None, order=3, mode='constant', cval=0.0, prefilter=True, chunklen=None, threads=None): """ Parallalized version of `scipy.ndimage.map_coordinates`. `scipy.ndimage.map_coordinates` is slow for large datasets. Speed improvement can be achieved by * Splitting the data into chunks * Performing the transformation of chunks in parallel New parameters: chunklen: Size of the chunks in pixels per axis. Default: None Special values: None: Automatic (Chooses a default based on number of dimensions) 0: Do not split data into chunks. (implicitly sets threads==1) threads: Number of threads. Default: None None: Automatic (One thread per available processing unit) """ # this part is taken without change from scipy's serial implementation if order < 0 or order > 5: raise RuntimeError('spline order not supported') input = np.asarray(input) if np.iscomplexobj(input): raise TypeError('Complex type not supported') coordinates = np.asarray(coordinates) if np.iscomplexobj(coordinates): raise TypeError('Complex type not supported') output_shape = coordinates.shape[1:] if input.ndim < 1 or len(output_shape) < 1: raise RuntimeError('input and output rank must be > 0') if coordinates.shape[0] != input.ndim: raise RuntimeError('invalid shape for coordinate array') mode = _ni_support._extend_mode_to_code(mode) if prefilter and order > 1: filtered = spline_filter(input, order, output=np.float64) else: filtered = input # return value of `_ni_support._get_output` changed between scipy versions, code here is # adapted to work with both output = _ni_support._get_output(output, input, shape=output_shape) retval = output if isinstance(output, tuple): output, retval = output # below here there is the new code for splitting into chunks and parallel execution if chunklen is None: # set defaults chunklen = 128 if output.ndim < 3: chunklen = 1024 def chunk_arguments(filtered, coordinates, output): chunks = [] for axis in range(output.ndim): chunkstarts = np.arange(0, output.shape[axis], chunklen) chunkends = chunkstarts + chunklen chunkends[-1] = output.shape[axis] chunks.append([slice(start, stop) for start, stop in zip(chunkstarts, chunkends)]) for chunk in itertools.product(*chunks): sub_coordinates = coordinates[(slice(None),) + chunk].copy() filtered_region = [] for in_axis in range(filtered.ndim): c = sub_coordinates[in_axis, ...] cmin = max(0, int(np.floor(np.min(c)))-5) cmax = min(filtered.shape[in_axis], int(np.ceil(np.max(c)))+5) sub_coordinates[in_axis, ...] -= cmin filtered_region.append(slice(cmin, cmax)) sub_filtered = filtered[tuple(filtered_region)] sub_output = output[chunk] yield (sub_filtered, sub_coordinates, sub_output) def map_coordinates_chunk(arg): sub_filtered, sub_coordinates, sub_output = arg _nd_image.geometric_transform(sub_filtered, None, sub_coordinates, None, None, sub_output, order, mode, cval, None, None) if chunklen > 0: list_of_chunk_args = list(chunk_arguments(filtered, coordinates, output)) else: list_of_chunk_args = [(filtered, coordinates, output)] if len(list_of_chunk_args) == 1: threads = 1 if threads != 1: threadpool = ThreadPoolExecutor(threads) my_map = threadpool.map else: my_map = map # execution happens here list(my_map(map_coordinates_chunk, list_of_chunk_args)) if threads != 1: if have_concurrent_futures: threadpool.shutdown() else: threadpool.close() threadpool.join() return retval
def map_coordinates_parallel(input, coordinates, output=None, order=3, mode='constant', cval=0.0, prefilter=True, chunklen=None, threads=None): """ Parallalized version of `scipy.ndimage.map_coordinates`. `scipy.ndimage.map_coordinates` is slow for large datasets. Speed improvement can be achieved by * Splitting the data into chunks * Performing the transformation of chunks in parallel New parameters: chunklen: Size of the chunks in pixels per axis. Default: None Special values: None: Automatic (Chooses a default based on number of dimensions) 0: Do not split data into chunks. (implicitly sets threads==1) threads: Number of threads. Default: None None: Automatic (One thread per available processing unit) """ # this part is taken without change from scipy's serial implementation if order < 0 or order > 5: raise RuntimeError('spline order not supported') input = np.asarray(input) if np.iscomplexobj(input): raise TypeError('Complex type not supported') coordinates = np.asarray(coordinates) if np.iscomplexobj(coordinates): raise TypeError('Complex type not supported') output_shape = coordinates.shape[1:] if input.ndim < 1 or len(output_shape) < 1: raise RuntimeError('input and output rank must be > 0') if coordinates.shape[0] != input.ndim: raise RuntimeError('invalid shape for coordinate array') mode = _ni_support._extend_mode_to_code(mode) if prefilter and order > 1: filtered = spline_filter(input, order, output=np.float64) else: filtered = input # return value of `_ni_support._get_output` changed between scipy versions, code here is # adapted to work with both output = _ni_support._get_output(output, input, shape=output_shape) retval = output if isinstance(output, tuple): output, retval = output # below here there is the new code for splitting into chunks and parallel execution if chunklen is None: # set defaults chunklen = 128 if output.ndim < 3: chunklen = 1024 def chunk_arguments(filtered, coordinates, output): chunks = [] for axis in range(output.ndim): chunkstarts = np.arange(0, output.shape[axis], chunklen) chunkends = chunkstarts + chunklen chunkends[-1] = output.shape[axis] chunks.append([ slice(start, stop) for start, stop in zip(chunkstarts, chunkends) ]) for chunk in itertools.product(*chunks): sub_coordinates = coordinates[(slice(None), ) + chunk].copy() filtered_region = [] for in_axis in range(filtered.ndim): c = sub_coordinates[in_axis, ...] cmin = max(0, int(np.floor(np.min(c))) - 5) cmax = min(filtered.shape[in_axis], int(np.ceil(np.max(c))) + 5) sub_coordinates[in_axis, ...] -= cmin filtered_region.append(slice(cmin, cmax)) sub_filtered = filtered[tuple(filtered_region)] sub_output = output[chunk] yield (sub_filtered, sub_coordinates, sub_output) def map_coordinates_chunk(arg): sub_filtered, sub_coordinates, sub_output = arg _nd_image.geometric_transform(sub_filtered, None, sub_coordinates, None, None, sub_output, order, mode, cval, None, None) if chunklen > 0: list_of_chunk_args = list( chunk_arguments(filtered, coordinates, output)) else: list_of_chunk_args = [(filtered, coordinates, output)] if len(list_of_chunk_args) == 1: threads = 1 if threads != 1: threadpool = ThreadPoolExecutor(threads) my_map = threadpool.map else: my_map = map # execution happens here list(my_map(map_coordinates_chunk, list_of_chunk_args)) if threads != 1: if have_concurrent_futures: threadpool.shutdown() else: threadpool.close() threadpool.join() return retval