Beispiel #1
0
def map_coordinates_parallel(input, coordinates, output=None, order=3, mode='constant', cval=0.0,
                             prefilter=True, chunklen=None, threads=None):
    """

    Parallalized version of `scipy.ndimage.map_coordinates`.

    `scipy.ndimage.map_coordinates` is slow for large datasets. Speed improvement can be
    achieved by

     * Splitting the data into chunks
     * Performing the transformation of chunks in parallel

    New parameters:

    chunklen: Size of the chunks in pixels per axis. Default: None
        Special values:
            None: Automatic (Chooses a default based on number of dimensions)
            0: Do not split data into chunks. (implicitly sets threads==1)

    threads: Number of threads. Default: None
        None: Automatic (One thread per available processing unit)

    """

    # this part is taken without change from scipy's serial implementation
    if order < 0 or order > 5:
        raise RuntimeError('spline order not supported')
    input = np.asarray(input)
    if np.iscomplexobj(input):
        raise TypeError('Complex type not supported')
    coordinates = np.asarray(coordinates)
    if np.iscomplexobj(coordinates):
        raise TypeError('Complex type not supported')
    output_shape = coordinates.shape[1:]
    if input.ndim < 1 or len(output_shape) < 1:
        raise RuntimeError('input and output rank must be > 0')
    if coordinates.shape[0] != input.ndim:
        raise RuntimeError('invalid shape for coordinate array')
    mode = _ni_support._extend_mode_to_code(mode)
    if prefilter and order > 1:
        filtered = spline_filter(input, order, output=np.float64)
    else:
        filtered = input

    # return value of `_ni_support._get_output` changed between scipy versions, code here is
    # adapted to work with both
    output = _ni_support._get_output(output, input, shape=output_shape)
    retval = output
    if isinstance(output, tuple):
        output, retval = output

    # below here there is the new code for splitting into chunks and parallel execution
    if chunklen is None:
        # set defaults
        chunklen = 128
        if output.ndim < 3:
            chunklen = 1024

    def chunk_arguments(filtered, coordinates, output):
        chunks = []
        for axis in range(output.ndim):
            chunkstarts = np.arange(0, output.shape[axis], chunklen)
            chunkends = chunkstarts + chunklen
            chunkends[-1] = output.shape[axis]
            chunks.append([slice(start, stop) for start, stop in zip(chunkstarts, chunkends)])

        for chunk in itertools.product(*chunks):
            sub_coordinates = coordinates[(slice(None),) + chunk].copy()
            filtered_region = []
            for in_axis in range(filtered.ndim):
                c = sub_coordinates[in_axis, ...]
                cmin = max(0, int(np.floor(np.min(c)))-5)
                cmax = min(filtered.shape[in_axis], int(np.ceil(np.max(c)))+5)
                sub_coordinates[in_axis, ...] -= cmin
                filtered_region.append(slice(cmin, cmax))
            sub_filtered = filtered[tuple(filtered_region)]
            sub_output = output[chunk]

            yield (sub_filtered, sub_coordinates, sub_output)

    def map_coordinates_chunk(arg):
        sub_filtered, sub_coordinates, sub_output = arg
        _nd_image.geometric_transform(sub_filtered, None, sub_coordinates, None, None,
                                      sub_output, order, mode, cval, None, None)

    if chunklen > 0:
        list_of_chunk_args = list(chunk_arguments(filtered, coordinates, output))
    else:
        list_of_chunk_args = [(filtered, coordinates, output)]

    if len(list_of_chunk_args) == 1:
        threads = 1

    if threads != 1:
        threadpool = ThreadPoolExecutor(threads)
        my_map = threadpool.map
    else:
        my_map = map

    # execution happens here
    list(my_map(map_coordinates_chunk, list_of_chunk_args))

    if threads != 1:
        if have_concurrent_futures:
            threadpool.shutdown()
        else:
            threadpool.close()
            threadpool.join()

    return retval
Beispiel #2
0
    def blocks(self,
               start=None,
               stop=None,
               max_batch_size=None,
               threading=False,
               thread_num=8,
               only_ops=False,
               only_virtual_ops=False):
        """ Yields blocks starting from ``start``.

            :param int start: Starting block
            :param int stop: Stop at this block
            :param int max_batch_size: only for appbase nodes. When not None, batch calls of are used.
                Cannot be combined with threading
            :param bool threading: Enables threading. Cannot be combined with batch calls
            :param int thread_num: Defines the number of threads, when `threading` is set.
            :param bool only_ops: Only yield operations (default: False).
                Cannot be combined with ``only_virtual_ops=True``.
            :param bool only_virtual_ops: Only yield virtual operations (default: False)

            .. note:: If you want instant confirmation, you need to instantiate
                      class:`beem.blockchain.Blockchain` with
                      ``mode="head"``, otherwise, the call will wait until
                      confirmed in an irreversible block.

        """
        # Let's find out how often blocks are generated!
        current_block = self.get_current_block()
        current_block_num = current_block.block_num
        if not start:
            start = current_block_num
        head_block_reached = False
        if threading and FUTURES_MODULE is not None:
            pool = ThreadPoolExecutor(max_workers=thread_num)
        elif threading:
            pool = Pool(thread_num, batch_mode=True)
        if threading:
            steem_instance = [self.steem]
            nodelist = self.steem.rpc.nodes.export_working_nodes()
            for i in range(thread_num - 1):
                steem_instance.append(
                    stm.Steem(node=nodelist,
                              num_retries=self.steem.rpc.num_retries,
                              num_retries_call=self.steem.rpc.num_retries_call,
                              timeout=self.steem.rpc.timeout))
        # We are going to loop indefinitely
        latest_block = 0
        while True:
            if stop:
                head_block = stop
            else:
                current_block_num = self.get_current_block_num()
                head_block = current_block_num
            if threading and not head_block_reached:
                latest_block = start - 1
                result_block_nums = []
                for blocknum in range(start, head_block + 1, thread_num):
                    # futures = []
                    i = 0
                    if FUTURES_MODULE is not None:
                        futures = []
                    block_num_list = []
                    # freeze = self.steem.rpc.nodes.freeze_current_node
                    num_retries = self.steem.rpc.nodes.num_retries
                    # self.steem.rpc.nodes.freeze_current_node = True
                    self.steem.rpc.nodes.num_retries = thread_num
                    error_cnt = self.steem.rpc.nodes.node.error_cnt
                    while i < thread_num and blocknum + i <= head_block:
                        block_num_list.append(blocknum + i)
                        results = []
                        if FUTURES_MODULE is not None:
                            futures.append(
                                pool.submit(Block,
                                            blocknum + i,
                                            only_ops=only_ops,
                                            only_virtual_ops=only_virtual_ops,
                                            steem_instance=steem_instance[i]))
                        else:
                            pool.enqueue(Block,
                                         blocknum + i,
                                         only_ops=only_ops,
                                         only_virtual_ops=only_virtual_ops,
                                         steem_instance=steem_instance[i])
                        i += 1
                    if FUTURES_MODULE is not None:
                        try:
                            results = [
                                r.result() for r in as_completed(futures)
                            ]
                        except Exception as e:
                            log.error(str(e))
                    else:
                        pool.run(True)
                        pool.join()
                        for result in pool.results():
                            results.append(result)
                        pool.abort()
                    self.steem.rpc.nodes.num_retries = num_retries
                    # self.steem.rpc.nodes.freeze_current_node = freeze
                    new_error_cnt = self.steem.rpc.nodes.node.error_cnt
                    self.steem.rpc.nodes.node.error_cnt = error_cnt
                    if new_error_cnt > error_cnt:
                        self.steem.rpc.nodes.node.error_cnt += 1
                    #    self.steem.rpc.next()

                    checked_results = []
                    for b in results:
                        if b.block_num is not None and int(
                                b.block_num) not in result_block_nums:
                            b["id"] = b.block_num
                            b.identifier = b.block_num
                            checked_results.append(b)
                            result_block_nums.append(int(b.block_num))

                    missing_block_num = list(
                        set(block_num_list).difference(set(result_block_nums)))
                    while len(missing_block_num) > 0:
                        for blocknum in missing_block_num:
                            try:
                                block = Block(
                                    blocknum,
                                    only_ops=only_ops,
                                    only_virtual_ops=only_virtual_ops,
                                    steem_instance=self.steem)
                                checked_results.append(block)
                                result_block_nums.append(int(block.block_num))
                            except Exception as e:
                                log.error(str(e))
                        missing_block_num = list(
                            set(block_num_list).difference(
                                set(result_block_nums)))
                    from operator import itemgetter
                    blocks = sorted(checked_results, key=itemgetter('id'))
                    for b in blocks:
                        if latest_block < int(b.block_num):
                            latest_block = int(b.block_num)
                        yield b

                if latest_block <= head_block:
                    for blocknum in range(latest_block + 1, head_block + 1):
                        if blocknum not in result_block_nums:
                            block = Block(blocknum,
                                          only_ops=only_ops,
                                          only_virtual_ops=only_virtual_ops,
                                          steem_instance=self.steem)
                            result_block_nums.append(blocknum)
                            yield block
            elif max_batch_size is not None and (
                    head_block -
                    start) >= max_batch_size and not head_block_reached:
                if not self.steem.is_connected():
                    raise OfflineHasNoRPCException(
                        "No RPC available in offline mode!")
                self.steem.rpc.set_next_node_on_empty_reply(False)
                latest_block = start - 1
                batches = max_batch_size
                for blocknumblock in range(start, head_block + 1, batches):
                    # Get full block
                    if (head_block - blocknumblock) < batches:
                        batches = head_block - blocknumblock + 1
                    for blocknum in range(blocknumblock,
                                          blocknumblock + batches - 1):
                        if only_virtual_ops:
                            if self.steem.rpc.get_use_appbase():
                                # self.steem.rpc.get_ops_in_block({"block_num": blocknum, 'only_virtual': only_virtual_ops}, api="account_history", add_to_queue=True)
                                self.steem.rpc.get_ops_in_block(
                                    blocknum,
                                    only_virtual_ops,
                                    add_to_queue=True)
                            else:
                                self.steem.rpc.get_ops_in_block(
                                    blocknum,
                                    only_virtual_ops,
                                    add_to_queue=True)
                        else:
                            if self.steem.rpc.get_use_appbase():
                                self.steem.rpc.get_block(
                                    {"block_num": blocknum},
                                    api="block",
                                    add_to_queue=True)
                            else:
                                self.steem.rpc.get_block(blocknum,
                                                         add_to_queue=True)
                        latest_block = blocknum
                    if batches >= 1:
                        latest_block += 1
                    if latest_block <= head_block:
                        if only_virtual_ops:
                            if self.steem.rpc.get_use_appbase():
                                # self.steem.rpc.get_ops_in_block({"block_num": blocknum, 'only_virtual': only_virtual_ops}, api="account_history", add_to_queue=False)
                                block_batch = self.steem.rpc.get_ops_in_block(
                                    blocknum,
                                    only_virtual_ops,
                                    add_to_queue=False)
                            else:
                                block_batch = self.steem.rpc.get_ops_in_block(
                                    blocknum,
                                    only_virtual_ops,
                                    add_to_queue=False)
                        else:
                            if self.steem.rpc.get_use_appbase():
                                block_batch = self.steem.rpc.get_block(
                                    {"block_num": latest_block},
                                    api="block",
                                    add_to_queue=False)
                            else:
                                block_batch = self.steem.rpc.get_block(
                                    latest_block, add_to_queue=False)
                        if not bool(block_batch):
                            raise BatchedCallsNotSupported()
                        blocknum = latest_block - len(block_batch) + 1
                        if not isinstance(block_batch, list):
                            block_batch = [block_batch]
                        for block in block_batch:
                            if not bool(block):
                                continue
                            if self.steem.rpc.get_use_appbase():
                                if only_virtual_ops:
                                    block = block["ops"]
                                else:
                                    block = block["block"]
                            block = Block(block,
                                          only_ops=only_ops,
                                          only_virtual_ops=only_virtual_ops,
                                          steem_instance=self.steem)
                            block["id"] = block.block_num
                            block.identifier = block.block_num
                            yield block
                            blocknum = block.block_num
            else:
                # Blocks from start until head block
                for blocknum in range(start, head_block + 1):
                    # Get full block
                    block = self.wait_for_and_get_block(
                        blocknum,
                        only_ops=only_ops,
                        only_virtual_ops=only_virtual_ops,
                        block_number_check_cnt=5,
                        last_current_block_num=current_block_num)
                    yield block
            # Set new start
            start = head_block + 1
            head_block_reached = True

            if stop and start > stop:
                return

            # Sleep for one block
            time.sleep(self.block_interval)
Beispiel #3
0
def map_coordinates_parallel(input,
                             coordinates,
                             output=None,
                             order=3,
                             mode='constant',
                             cval=0.0,
                             prefilter=True,
                             chunklen=None,
                             threads=None):
    """

    Parallalized version of `scipy.ndimage.map_coordinates`.

    `scipy.ndimage.map_coordinates` is slow for large datasets. Speed improvement can be
    achieved by

     * Splitting the data into chunks
     * Performing the transformation of chunks in parallel

    New parameters:

    chunklen: Size of the chunks in pixels per axis. Default: None
        Special values:
            None: Automatic (Chooses a default based on number of dimensions)
            0: Do not split data into chunks. (implicitly sets threads==1)

    threads: Number of threads. Default: None
        None: Automatic (One thread per available processing unit)

    """

    # this part is taken without change from scipy's serial implementation
    if order < 0 or order > 5:
        raise RuntimeError('spline order not supported')
    input = np.asarray(input)
    if np.iscomplexobj(input):
        raise TypeError('Complex type not supported')
    coordinates = np.asarray(coordinates)
    if np.iscomplexobj(coordinates):
        raise TypeError('Complex type not supported')
    output_shape = coordinates.shape[1:]
    if input.ndim < 1 or len(output_shape) < 1:
        raise RuntimeError('input and output rank must be > 0')
    if coordinates.shape[0] != input.ndim:
        raise RuntimeError('invalid shape for coordinate array')
    mode = _ni_support._extend_mode_to_code(mode)
    if prefilter and order > 1:
        filtered = spline_filter(input, order, output=np.float64)
    else:
        filtered = input

    # return value of `_ni_support._get_output` changed between scipy versions, code here is
    # adapted to work with both
    output = _ni_support._get_output(output, input, shape=output_shape)
    retval = output
    if isinstance(output, tuple):
        output, retval = output

    # below here there is the new code for splitting into chunks and parallel execution
    if chunklen is None:
        # set defaults
        chunklen = 128
        if output.ndim < 3:
            chunklen = 1024

    def chunk_arguments(filtered, coordinates, output):
        chunks = []
        for axis in range(output.ndim):
            chunkstarts = np.arange(0, output.shape[axis], chunklen)
            chunkends = chunkstarts + chunklen
            chunkends[-1] = output.shape[axis]
            chunks.append([
                slice(start, stop)
                for start, stop in zip(chunkstarts, chunkends)
            ])

        for chunk in itertools.product(*chunks):
            sub_coordinates = coordinates[(slice(None), ) + chunk].copy()
            filtered_region = []
            for in_axis in range(filtered.ndim):
                c = sub_coordinates[in_axis, ...]
                cmin = max(0, int(np.floor(np.min(c))) - 5)
                cmax = min(filtered.shape[in_axis],
                           int(np.ceil(np.max(c))) + 5)
                sub_coordinates[in_axis, ...] -= cmin
                filtered_region.append(slice(cmin, cmax))
            sub_filtered = filtered[tuple(filtered_region)]
            sub_output = output[chunk]

            yield (sub_filtered, sub_coordinates, sub_output)

    def map_coordinates_chunk(arg):
        sub_filtered, sub_coordinates, sub_output = arg
        _nd_image.geometric_transform(sub_filtered, None, sub_coordinates,
                                      None, None, sub_output, order, mode,
                                      cval, None, None)

    if chunklen > 0:
        list_of_chunk_args = list(
            chunk_arguments(filtered, coordinates, output))
    else:
        list_of_chunk_args = [(filtered, coordinates, output)]

    if len(list_of_chunk_args) == 1:
        threads = 1

    if threads != 1:
        threadpool = ThreadPoolExecutor(threads)
        my_map = threadpool.map
    else:
        my_map = map

    # execution happens here
    list(my_map(map_coordinates_chunk, list_of_chunk_args))

    if threads != 1:
        if have_concurrent_futures:
            threadpool.shutdown()
        else:
            threadpool.close()
            threadpool.join()

    return retval
Beispiel #4
0
    while True:
        os.system("clear")
        time.sleep(10)
if __name__ == '__main__':

    lock = threading.Lock() #初始化一把锁
    pool = ThreadPoolExecutor(max_workers=5)
    try:
        with open("./config.yaml", "r",encoding='utf-8') as yaml_file:
            yaml_obj = yaml.load(yaml_file.read())
        
        USERNAME = yaml_obj['USERNAME']
        PASSWORD = yaml_obj['PASSWORD']

        for platform in list(yaml_obj.keys())[2:]:
            for massage in yaml_obj[platform]:
                achor_name = massage
                upload_platform = ''
                if len(yaml_obj[platform][massage])==2:
                    rid = yaml_obj[platform][massage][0]
                    upload_platform = yaml_obj[platform][massage][1]
                else:rid = yaml_obj[platform][massage][0]
                pool.submit(download, achor_name,rid,platform,upload_platform,USERNAME,PASSWORD,lock)
        pool.submit(clearshell)
        pool.close()
        pool.join()
    except Exception as err:
        print(err)
        time.sleep(10)
            
Beispiel #5
0
        else:
            path += '.png'
        download(path, url)
    return


def download(path, url):
    try:
        photodata = requests.get(url).content
        with open(r'bqb\%s' % path, 'wb') as w:
            w.write(photodata)
        print(path + '下载完成')
        return
    except:
        print(path + "下载错误!")
        return


if __name__ == "__main__":
    start = time.time()
    urllist = [
        'https://www.fabiaoqing.com/biaoqing/lists/page/%d.html' % i
        for i in range(1, 5)
    ]
    pool = ThreadPoolExecutor(4)
    while len(urllist):
        pool.submit(Data, urllist[0])
        pool.join(2)
        urllist.pop(0)
    print("耗时:", time.time() - start)