Example #1
0
 def test_apply_async(self):
     done = Event()
     def some_work(x):
         done.set()
     pool = self.klass(2)
     pool.apply_async(some_work, ('x', ))
     done.wait()
Example #2
0
 def test_reentrant(self):
     pool = self.klass(1)
     result = pool.apply(pool.apply, (lambda a: a + 1, (5, )))
     self.assertEqual(result, 6)
     evt = Event()
     pool.apply_async(evt.set)
     evt.wait()
Example #3
0
    def test_apply_async(self):
        done = Event()

        def some_work(x):
            done.set()

        pool = self.klass(2)
        pool.apply_async(some_work, ('x', ))
        done.wait()
Example #4
0
    def fire_async(self, args=(), kwargs={}, concurrency=None):
        """Fire the signal asynchronously - each callback in its own greenlet.

        If concurrency is given, it is the limit on the number of dispatching
        greenlets to execute concurrently.
        """
        pool = gevent.pool.Pool(size=concurrency)
        for callback in self.callback.values():
            pool.apply_async(callback, args, kwargs)
Example #5
0
 def _polling_get_status(self, pool):
     sender = self.target.name
     logger.debug("POLLING: sender={} signal={}".format(
         sender, self.signal))
     if self.signal.has_receivers_for(sender):
         logger.debug("POLLING: target={}".format(self.target.nodes))
         for cnode in self.target.nodes:
             pool.apply_async(
                 self.get_status, [[cnode]],
                 callback=lambda x: self.signal.send(sender, message=x))
     else:
         logger.debug("no POLLING: sender={} receiver={}".format(
             sender, self.signal.receivers))
Example #6
0
 def _polling_get_status(self, pool):
     sender = self.target.name
     logger.debug("POLLING: sender={} signal={}".format(
         sender, self.signal))
     if self.signal.has_receivers_for(sender):
         logger.debug("POLLING: target={}".format(self.target.nodes))
         for cnode in self.target.nodes:
             pool.apply_async(
                 self.get_status, [[cnode]],
                 callback=lambda x: self.signal.send(sender, message=x))
     else:
         logger.debug("no POLLING: sender={} receiver={}".format(
             sender, self.signal.receivers))
Example #7
0
def jobDispatcher(q):
    resultsFolder = Configs().get('resultsFolder')
    xputBuckets = Configs().get('xputBuckets')
    alpha = Configs().get('alpha')
    pool = gevent.pool.Pool()
    while True:
        args = q.get()
        pool.apply_async(analyzer,
                         args=(
                             args,
                             resultsFolder,
                             xputBuckets,
                             alpha,
                         ))
Example #8
0
    def test_reentrant(self):
        pool = self.klass(1)
        def reenter():
            result = pool.apply(lambda a: a, ('reenter', ))
            self.assertEqual('reenter', result)

        pool.apply(reenter)

        evt = Event()
        def reenter_async():
            pool.apply_async(lambda a: a, ('reenter', ))
            evt.set()

        pool.apply_async(reenter_async)
        evt.wait()
Example #9
0
def poi_ori(poi_type):
    already_merged_city = init_already_merged_city(poi_type=poi_type)
    conn = base_data_pool.connection()
    cursor = conn.cursor()
    cursor.execute('''SELECT id
FROM city;''')
    cids = list(map(lambda x: x[0], cursor.fetchall()))
    cursor.close()
    conn.close()
    for cid in cids:
        if cid in already_merged_city:
            continue
        start = time.time()
        logger.info('[start][cid: {}]'.format(cid))
        pool.apply_async(poi_merge, args=(cid, poi_type))
        logger.info('[end][cid: {}][takes: {}]'.format(cid, time.time() - start))
    pool.join()
Example #10
0
    def test_reentrant(self):
        pool = self.klass(1)

        def reenter():
            result = pool.apply(lambda a: a, ('reenter', ))
            self.assertEqual('reenter', result)

        pool.apply(reenter)

        evt = Event()

        def reenter_async():
            pool.apply_async(lambda a: a, ('reenter', ))
            evt.set()

        pool.apply_async(reenter_async)
        evt.wait()
Example #11
0
    def test_multiple_coros(self):
        evt = Event()
        results = []
        def producer():
            results.append('prod')
            evt.set()

        def consumer():
            results.append('cons1')
            evt.wait()
            results.append('cons2')

        pool = self.klass(2)
        done = pool.spawn(consumer)
        pool.apply_async(producer)
        done.get()
        self.assertEquals(['cons1', 'prod', 'cons2'], results)
def download_pic():
    conn = pymysql.connect(host='10.10.228.253', user='******', password='******', charset='utf8',
                           db='ServicePlatform')
    cursor = conn.cursor()
    cursor.execute("SELECT file_name, source, sid FROM images_attr_daodao_20170929a WHERE `use`=1 LIMIT 100;")
    start = time.time()
    _count = 0
    for file_name, source, sid in cursor.fetchall():
        _count += 1
        parent_path = os.path.join(PARENT_PATH, "###".join([source, sid]))
        if not os.path.exists(parent_path):
            os.makedirs(parent_path)
        pool.apply_async(download, ("mioji-attr", file_name, parent_path))
    pool.join()
    cursor.close()
    conn.close()
    print("[Total: {}][Takes: {}]".format(_count, time.time() - start))
Example #13
0
    def test_multiple_coros(self):
        evt = Event()
        results = []

        def producer():
            results.append('prod')
            evt.set()

        def consumer():
            results.append('cons1')
            evt.wait()
            results.append('cons2')

        pool = self.klass(2)
        done = pool.spawn(consumer)
        pool.apply_async(producer)
        done.get()
        self.assertEquals(['cons1', 'prod', 'cons2'], results)
Example #14
0
    def test_multiple_coros(self):
        evt = Event()
        results = []

        def producer():
            gevent.sleep(0.001)
            results.append("prod")
            evt.set()

        def consumer():
            results.append("cons1")
            evt.wait()
            results.append("cons2")

        pool = self.klass(2)
        done = pool.spawn(consumer)
        pool.apply_async(producer)
        done.get()
        self.assertEqual(["cons1", "prod", "cons2"], results)
Example #15
0
def startBruteforce(sClient, pool, wordlist):
    for word in open(wordlist, "r"):
        pool.add(
            pool.apply_async(
                isSecretSaslValid,
                args=(sClient, word.strip(), "sparkSaslUser", True),
                callback=checkResult,
            ))

    pool.join(timeout=30)
    pool.kill()
    whine("Could not find the secret", "warn")
Example #16
0
    def task_run(self):
        pool = gevent.pool.Pool(2)

        pool.apply_async(self.task_all,
                         args=(self.worker_list_new, self.webpage_db_new,
                               CHOOSE_DB_NEW))

        pool.apply_async(self.task_all,
                         args=(self.worker_list_old, self.webpage_db_old,
                               CHOOSE_DB_OLD))

        self.log.info("所有任务加载完成..")
        pool.join()

        current_time = 0
        sleep_time = 300
        self.log.info('完成解析!!, 开始休眠休眠时间为: {rand}'.format(rand=sleep_time))

        while is_running and current_time < 300:
            time.sleep(1)
            current_time += 1
Example #17
0
    def _s3_upload_pg_archive(self,
                              archive_filename,
                              pool_size,
                              rate_limit=None):
        """
        Upload archive_filename to s3_url_prefix.
        """
        backup_s3_prefix = ('{0}/basebackups_{1}/base_{2}'.format(
            self.s3_prefix, FILE_STRUCTURE_VERSION,
            os.path.basename(archive_filename)))

        # absolute upload paths are used for telling lzop what to compress
        local_abspath = os.path.abspath(archive_filename)

        partitions = tar_partition.archive_partitions_plan(
            local_abspath,

            # 1610612736 bytes == 1.5 gigabytes, per partition,
            # non-tunable
            1610612736)

        if rate_limit is None:
            per_process_limit = None
        else:
            per_process_limit = int(rate_limit / pool_size)

        # Reject tiny per-process rate limits.  They should be
        # rejected more nicely elsewhere.
        assert per_process_limit > 0 or per_process_limit is None

        # a list to accumulate async upload jobs
        uploads = []

        total_size = os.path.getsize(local_abspath)

        pool = gevent.pool.Pool(size=pool_size)

        # Enqueue uploads for parallel execution
        try:
            for part in partitions:
                uploads.append(
                    pool.apply_async(s3_worker.do_archive_partition_put, [
                        backup_s3_prefix, part, per_process_limit,
                        self.gpg_key_id
                    ]))
        finally:
            while uploads:
                uploads.pop().get()

            pool.join()

        return backup_s3_prefix, total_size
Example #18
0
        def _validate_proxy_list(self, proxies, timeout=90):
            valid_proxies = []

            def save_result(p):
                if p and len(str(p)):
                    valid_proxies.append(p)

            # 在多进程下面, 只检测部分
            if self.process_count > 1:
                num_per_part = int((len(proxies) + self.process_count - 1) /
                                   self.process_count)
                start = self.process_seq * num_per_part
                end = (self.process_seq + 1) * num_per_part
                proxies = proxies[start:end]

            # 每100 检查一次
            total_num = len(proxies)
            step = 500
            pool = gevent.pool.Pool(500)
            for i in range(0, total_num, step):
                group = proxies[i:i + step]
                for proxy in group:
                    pool.apply_async(self._validate_one_proxy,
                                     args=(proxy, 'http'),
                                     callback=save_result)
                    pool.apply_async(self._validate_one_proxy,
                                     args=(proxy, 'https'),
                                     callback=save_result)
                    pool.apply_async(self._validate_one_proxy,
                                     args=(proxy, 'socks4'),
                                     callback=save_result)
                    pool.apply_async(self._validate_one_proxy,
                                     args=(proxy, 'socks5'),
                                     callback=save_result)

                is_empty = pool.join(timeout=timeout)
                if not is_empty:
                    # 还没有检查完整
                    print("**** validation is not done!")

                pool.kill()

                # 即时保存检验过的traffic
                self.save_proxies(valid_proxies)
                valid_proxies = []
                print("progress: {}/{}".format(i, total_num))
                time.sleep(1)

            return valid_proxies
Example #19
0
def pint_test(config):
    # 创建进程间通信队列
    activeq = Queue()
    notactiveq = Queue()
    host_list = []
    ips = config["ip"].split("-")
    start_ip = ips[0].split('.')
    end_ip = ips[1].split('.')
    base_ip = '.'.join(start_ip[:3])
    start_num = start_ip[3]
    end_num = end_ip[3]
    for ipnum in range(int(start_num), int(end_num)):
        host_list.append(f'{base_ip}.{str(ipnum)}')
    # 创建进程池
    pool = Pool(processes=int(config['n']))
    for ipaddr in host_list:
        pool.apply_async(ping_host, args=(activeq, notactiveq, ipaddr))
    pool.close()
    pool.join()
    # 输出正在使用ip
    read(activeq)
    # 输出未被使用ip
    read(notactiveq)
Example #20
0
    def _s3_upload_pg_archive(self, archive_filename, pool_size,
                              rate_limit=None):
        """
        Upload archive_filename to s3_url_prefix.
        """
        backup_s3_prefix = ('{0}/basebackups_{1}/base_{2}'
                            .format(self.s3_prefix, FILE_STRUCTURE_VERSION,
                                    os.path.basename(archive_filename)))

        # absolute upload paths are used for telling lzop what to compress
        local_abspath = os.path.abspath(archive_filename)

        partitions = tar_partition.archive_partitions_plan(local_abspath,

            # 1610612736 bytes == 1.5 gigabytes, per partition,
            # non-tunable
            1610612736)

        if rate_limit is None:
            per_process_limit = None
        else:
            per_process_limit = int(rate_limit / pool_size)

        # Reject tiny per-process rate limits.  They should be
        # rejected more nicely elsewhere.
        assert per_process_limit > 0 or per_process_limit is None

        # a list to accumulate async upload jobs
        uploads = []

        total_size = os.path.getsize(local_abspath)

        pool = gevent.pool.Pool(size=pool_size)

        # Enqueue uploads for parallel execution
        try:
            for part in partitions:
                uploads.append(pool.apply_async(
                        s3_worker.do_archive_partition_put,
                        [backup_s3_prefix, part, per_process_limit,
                         self.gpg_key_id]))
        finally:
            while uploads:
                uploads.pop().get()

            pool.join()

        return backup_s3_prefix, total_size
Example #21
0
def keep_fqsocks_busy():
    while True:
        pool = gevent.pool.Pool(size=16)
        greenlets = []
        for i in range(100):
            greenlets.append(pool.apply_async(check_twitter_access))
        while len(pool) > 0:
            for greenlet in list(pool):
                try:
                    greenlet.join(timeout=10)
                except:
                    pass
        try:
            pool.kill()
        except:
            pass
Example #22
0
def keep_fqsocks_busy():
    while True:
        pool = gevent.pool.Pool(size=16)
        greenlets = []
        for i in range(100):
            greenlets.append(pool.apply_async(check_twitter_access))
        while len(pool) > 0:
            for greenlet in list(pool):
                try:
                    greenlet.join(timeout=10)
                except:
                    pass
        try:
            pool.kill()
        except:
            pass
Example #23
0
def keep_fqsocks_busy():
    goagent.GoAgentProxy.GOOGLE_HOSTS = ["goagent-google-ip.fqrouter.com"]
    goagent.GoAgentProxy.refresh(fqsocks.mandatory_proxies)
    while True:
        pool = gevent.pool.Pool(size=16)
        greenlets = []
        for i in range(100):
            greenlets.append(pool.apply_async(check_baidu_access))
        while len(pool) > 0:
            for greenlet in list(pool):
                try:
                    greenlet.join(timeout=10)
                except:
                    pass
        try:
            pool.kill()
        except:
            pass
def getPingResult(src_ip, count):
    """
    当前的ping往返平均时间 oid为:  1.3.6.1.2.1.80.1.3.1.6
    1.获取交换机所有nqa实例名
    2.获取交换机nqa实例对应ping的平均时间(ms)
    :return:
    """
    # 获取交换机所有nqa实例名,列表中为字典格式name:dst_ip
    test_name_list = getTestName(src_ip)
    # 获取交换机nqa实例对应ping的平均时间(ms),测试时间
    print "test_name_list: %s" % test_name_list
    pool = mp.Pool(processes=count)
    result_tmp = [
        pool.apply_async(process_start, args=(src_ip, test_name[0]))
        for test_name in test_name_list
    ]
    ping_result = [p.get() for p in result_tmp]
    print '*****', ping_result
    return ping_result
        }
    try:
        r = requests.get('http://store.nike.com/cn/zh_cn/', proxies=proxies, timeout=(3, 1))
    except requests.exceptions.ConnectionError:
        return
    except requests.exceptions.ReadTimeout:
        return
    end = time.time()
    delay = '{:.0f}ms'.format((end-start)*1000)
    queue.put([index, delay])

if __name__ == '__main__':
    with open('give.txt', 'r') as f:
        ips = f.read().strip().split('\n')
    pool = gevent.pool.Pool(len(ips))
    queue = gevent.queue.Queue()
    for index, ip in enumerate(ips):
        pool.apply_async(ip_delay, (index, ip))
    pool.join()

    # ip_delay(00, None)
    nums = []
    while True:
        if queue.qsize() > 0:
            task = queue.get()
            print(task)
            nums.append(task[0])
        else:
            break
    nums.sort()
    print(nums)
Example #26
0
    def _s3_upload_pg_cluster_dir(self, start_backup_info, pg_cluster_dir,
                                  version, pool_size, rate_limit=None):
        """
        Upload to s3_url_prefix from pg_cluster_dir

        This function ignores the directory pg_xlog, which contains WAL
        files and are not generally part of a base backup.

        Note that this is also lzo compresses the files: thus, the number
        of pooled processes involves doing a full sequential scan of the
        uncompressed Postgres heap file that is pipelined into lzo. Once
        lzo is completely finished (necessary to have access to the file
        size) the file is sent to S3.

        TODO: Investigate an optimization to decouple the compression and
        upload steps to make sure that the most efficient possible use of
        pipelining of network and disk resources occurs.  Right now it
        possible to bounce back and forth between bottlenecking on reading
        from the database block device and subsequently the S3 sending
        steps should the processes be at the same stage of the upload
        pipeline: this can have a very negative impact on being able to
        make full use of system resources.

        Furthermore, it desirable to overflowing the page cache: having
        separate tunables for number of simultanious compression jobs
        (which occupy /tmp space and page cache) and number of uploads
        (which affect upload throughput) would help.

        """
        parts = tar_partition.partition(pg_cluster_dir)

        backup_s3_prefix = ('{0}/basebackups_{1}/'
                            'base_{file_name}_{file_offset}'
                            .format(self.s3_prefix, FILE_STRUCTURE_VERSION,
                                    **start_backup_info))

        if rate_limit is None:
            per_process_limit = None
        else:
            per_process_limit = int(rate_limit / pool_size)

        # Reject tiny per-process rate limits.  They should be
        # rejected more nicely elsewhere.
        assert per_process_limit > 0 or per_process_limit is None

        # a list to accumulate async upload jobs
        uploads = []

        total_size = 0

        # Make an attempt to upload extended version metadata
        extended_version_url = backup_s3_prefix + '/extended_version.txt'
        logger.info(
            msg='start upload postgres version metadata',
            detail=('Uploading to {extended_version_url}.'
                    .format(extended_version_url=extended_version_url)))
        s3_worker.uri_put_file(extended_version_url, StringIO(version),
                               content_encoding='text/plain')
        logger.info(msg='postgres version metadata upload complete')

        pool = gevent.pool.Pool(size=pool_size)

        # Enqueue uploads for parallel execution
        try:
            for tpart in parts:
                total_size += tpart.total_member_size
                uploads.append(pool.apply_async(
                        s3_worker.do_partition_put,
                        [backup_s3_prefix, tpart, per_process_limit,
                         self.gpg_key_id]))
        finally:
            while uploads:
                uploads.pop().get()

            pool.join()

        return backup_s3_prefix, total_size
Example #27
0
 def wrapper(*args, **kwds):
     pool = kwds.pop('pool')
     return pool.apply_async(f, args=args, kwds=kwds)
Example #28
0
 def reenter_async():
     pool.apply_async(lambda a: a, ('reenter', ))
     evt.set()
Example #29
0
# @Time    : 2017/9/5 下午4:42
# @Author  : Hou Rong
# @Site    :
# @File    : test.py
# @Software: PyCharm
from gevent.monkey import patch_all

patch_all()
import gevent.pool

pool = gevent.pool.Pool()

from rate_limit_redis import RateLimitRedis
import redis
import time

redis_conn = redis.Redis()
key = 'hourong.me'


@RateLimitRedis(redis_conn=redis_conn, key=key, period=2)
def do_something(args):
    print("start", args, time.time())
    time.sleep(10)
    print("end", args, time.time())


for i in range(5):
    pool.apply_async(do_something, (i, ))
pool.join()
Example #30
0
 def _polling_get_cell_status(self, pool):
     sender = self.target.name
     if self.signal.has_receivers_for(sender):
         pool.apply_async(
             self.get_cell_status,
             callback=lambda x: self.signal.send(sender, message=x))
Example #31
0
    def fetch(self):
        pool = gevent.pool.Pool(size=6)
        for repo in self._repository_iter():
            pool.apply_async(repo.fetch, [self.dry_run])

        pool.join()
Example #32
0
    def _s3_upload_pg_cluster_dir(self, start_backup_info, pg_cluster_dir,
                                  version, pool_size, rate_limit=None):
        """
        Upload to s3_url_prefix from pg_cluster_dir

        This function ignores the directory pg_xlog, which contains WAL
        files and are not generally part of a base backup.

        Note that this is also lzo compresses the files: thus, the number
        of pooled processes involves doing a full sequential scan of the
        uncompressed Postgres heap file that is pipelined into lzo. Once
        lzo is completely finished (necessary to have access to the file
        size) the file is sent to S3.

        TODO: Investigate an optimization to decouple the compression and
        upload steps to make sure that the most efficient possible use of
        pipelining of network and disk resources occurs.  Right now it
        possible to bounce back and forth between bottlenecking on reading
        from the database block device and subsequently the S3 sending
        steps should the processes be at the same stage of the upload
        pipeline: this can have a very negative impact on being able to
        make full use of system resources.

        Furthermore, it desirable to overflowing the page cache: having
        separate tunables for number of simultanious compression jobs
        (which occupy /tmp space and page cache) and number of uploads
        (which affect upload throughput) would help.

        """

        # Get a manifest of files first.
        matches = []

        def raise_walk_error(e):
            raise e

        walker = os.walk(pg_cluster_dir, onerror=raise_walk_error)
        for root, dirnames, filenames in walker:
            is_cluster_toplevel = (os.path.abspath(root) ==
                                   os.path.abspath(pg_cluster_dir))

            # Do not capture any WAL files, although we do want to
            # capture the WAL directory or symlink
            if is_cluster_toplevel:
                if 'pg_xlog' in dirnames:
                    dirnames.remove('pg_xlog')
                    matches.append(os.path.join(root, 'pg_xlog'))

            for filename in filenames:
                if is_cluster_toplevel and filename in ('postmaster.pid',
                                                        'postgresql.conf'):
                    # Do not include the postmaster pid file or the
                    # configuration file in the backup.
                    pass
                else:
                    matches.append(os.path.join(root, filename))

            # Special case for empty directories
            if not filenames:
                matches.append(root)

        backup_s3_prefix = ('{0}/basebackups_{1}/'
                            'base_{file_name}_{file_offset}'
                            .format(self.s3_prefix, FILE_STRUCTURE_VERSION,
                                    **start_backup_info))

        # absolute upload paths are used for telling lzop what to compress
        local_abspaths = [os.path.abspath(match) for match in matches]

        # computed to subtract out extra extraneous absolute path
        # information when storing on S3
        common_local_prefix = os.path.commonprefix(local_abspaths)

        partitions = tar_partition.tar_partitions_plan(
            common_local_prefix, local_abspaths,

            # 1610612736 bytes == 1.5 gigabytes, per partition,
            # non-tunable
            1610612736)

        if rate_limit is None:
            per_process_limit = None
        else:
            per_process_limit = int(rate_limit / pool_size)

        # Reject tiny per-process rate limits.  They should be
        # rejected more nicely elsewhere.
        assert per_process_limit > 0 or per_process_limit is None

        # a list to accumulate async upload jobs
        uploads = []

        total_size = 0

        # Make an attempt to upload extended version metadata
        extended_version_url = backup_s3_prefix + '/extended_version.txt'
        logger.info(msg='start upload postgres version metadata',
                    detail=('Uploading to {extended_version_url}.'
                            .format(extended_version_url=extended_version_url)))
        s3_worker.uri_put_file(extended_version_url, StringIO(version),
                               content_encoding='text/plain')
        logger.info(msg='postgres version metadata upload complete')

        pool = gevent.pool.Pool(size=pool_size)

        # Enqueue uploads for parallel execution
        try:
            for tpart in partitions:
                total_size += tpart.total_member_size
                uploads.append(pool.apply_async(
                        s3_worker.do_partition_put,
                        [backup_s3_prefix, tpart, per_process_limit, self.gpg_key_id]))
        finally:
            while uploads:
                uploads.pop().get()

            pool.join()

        return backup_s3_prefix, total_size
Example #33
0
 def _polling_get_cell_status(self, pool):
     sender = self.target.name
     if self.signal.has_receivers_for(sender):
         pool.apply_async(
             self.get_cell_status,
             callback=lambda x: self.signal.send(sender, message=x))
Example #34
0
 def post(self):
     pool = self.application.settings.get('POSTpool')
     args = self.request.arguments
     LOG_ACTION(logger, 'POST:' + str(args))
     pool.apply_async(postHandler, (args, ), callback=self._callback)
Example #35
0
 def reenter_async():
     pool.apply_async(lambda a: a, ('reenter', ))
     evt.set()
Example #36
0
    def _download_res(
            self,
            filepath,
            rate,
            uploading=True,
            callback=None,
            cb_kwargs=None):
        try:
            peers = self.get_peers()
            self._record_get_peer_ts()
            peers_num = len(peers)
            count = 0

            # just get resource size
            while True:
                ip, port = peers[count]
                logger.info('get resource size')
                try:
                    ret = self._requests_session.get(
                        "{protocol}{ip}:{port}/?{res}"
                        .format(
                            protocol=PROTOCOL, ip=ip,
                            port=port,
                            res=urlencode({'res_url': self.upload_res_url})),
                        stream=True,
                        headers={"Range": "bytes=0-0"},
                        timeout=1)

                    if ret.ok:
                        #: bytes=0-1/17)
                        content_range = ret.headers.get("Content-Range")
                        res_length = content_range.split('/')[-1]
                        break
                    else:
                        logger.warn(
                            'get piece from ip: %s port: %s error, code: %s ' %
                            (ip, port, ret.status_code))
                        count += 1
                        self.del_from_tracker(ip=ip, peer_port=port)
                except ConnectionError:
                    logger.warn(
                        'get piece from ip: %s port: %s error ConnectionError'
                        % (ip, port))
                    count += 1
                    self.del_from_tracker(ip=ip, peer_port=port)
                except Timeout:
                    logger.warn(
                        'get piece from ip: %s port: %s error Timeout' %
                        (ip, port))
                    count += 1
                    self.del_from_tracker(ip=ip, peer_port=port)
                finally:
                    if count >= peers_num:
                        logger.warn("No peers avaliable")
                        peers = self.get_peers()
                        peers_num = len(peers)
                        count = 0

            logger.info('%s is size of %s' %
                        (self.upload_res_url, sizeof_fmt_human(res_length)))

            self.piece_file = PieceFile(res_length, filepath)

            pool_work_num = 15
            pool_q_size = pool_work_num * 2
            pool = gevent.pool.Pool(pool_work_num)
            self.start_ready_upload_thread()

            if rate:
                self.download_rate = rate
            else:
                rate = self.download_rate

            if rate:
                self.token_bucket = TokenBucket(rate)

            while self.piece_file.has_unalloc():
                args_list = list()
                for peer in peers:
                    if peer not in self._peer_in_conn:
                        args_list.append((peer, None))
                [pool.apply_async(self._download_piece_thread, *args)
                    for args in args_list[:pool_q_size]]
                # update peers if peer run out
                while pool.full():
                    gevent.sleep(0.2)

                if not self.piece_file.has_empty():
                    pool.join()

                logger.debug(
                    'test get_empty_block: %s' %
                    self.piece_file.get_empty_piece())

                logger.debug('peer in connection:  %s' % self._peer_in_conn)
                if self.piece_file.has_unalloc():
                    try:
                        tv = self._get_last_get_peer_tv()
                        if tv < GET_PEER_INTERVAL:
                            gevent.sleep(GET_PEER_INTERVAL - tv)
                        g = gevent.spawn(self.get_peers)
                        peers = g.get()
                        self._record_get_peer_ts()
                    except NoPeersFound:
                        # if pool.workRequests:
                        if pool_work_num - pool.free_count() > 0:
                            # some remained piece maybe on the way
                            pool.join()
                            if self.piece_file.has_unalloc():
                                tv = self._get_last_get_peer_tv()
                                if tv > GET_PEER_INTERVAL:
                                    gevent.sleep(GET_PEER_INTERVAL - tv)
                                g = gevent.spawn(self.get_peers)
                                peers = g.get()
                                self._record_get_peer_ts()
                            else:
                                break
                        else:
                            logger.error("no worker running, and get no peers")
                            raise
                else:
                    break

            logger.info('File is complete, size: %s' %
                        self.piece_file.get_real_filesize())

        except NoPeersFound:
            if self.fallback:
                logger.info('Use fallback way to get resouce')
                try:
                    res_length = get_res_length(self.res_url)
                except ConnectionError:
                    raise OriginURLConnectError(self.res_url)
                logger.info(
                    'get resource length %s' %
                    sizeof_fmt_human(res_length))
                if not self.piece_file:
                    self.piece_file = PieceFile(res_length, filepath)

                self.start_ready_upload_thread()
                http_download_to_piecefile(
                        self.res_url, self.piece_file)
            else:
                self.deactivate()
                raise

        # self.piece_file.tofile()
        self.res_is_downloaded = True
        if callback:
            logger.info('Run callback')
            callback(**cb_kwargs)
Example #37
0
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2017/10/31 上午9:39
# @Author  : Hou Rong
# @Site    :
# @File    : gevent_pool_timout_test.py
# @Software: PyCharm
import gevent.monkey

gevent.monkey.patch_all()
import gevent.pool
import time
import random

pool = gevent.pool.Pool(size=10)


def hello_world():
    t = random.randint(1, 10)
    time.sleep(t)
    print("Hello World, {}".format(t))


if __name__ == '__main__':
    for i in range(10):
        pool.apply_async(hello_world, ())
    pool.join(timeout=5)
Example #38
0
    def _s3_upload_pg_cluster_dir(self, start_backup_info, pg_cluster_dir,
                                  version, pool_size, rate_limit=None):
        """
        Upload to s3_url_prefix from pg_cluster_dir

        This function ignores the directory pg_xlog, which contains WAL
        files and are not generally part of a base backup.

        Note that this is also lzo compresses the files: thus, the number
        of pooled processes involves doing a full sequential scan of the
        uncompressed Postgres heap file that is pipelined into lzo. Once
        lzo is completely finished (necessary to have access to the file
        size) the file is sent to S3.

        TODO: Investigate an optimization to decouple the compression and
        upload steps to make sure that the most efficient possible use of
        pipelining of network and disk resources occurs.  Right now it
        possible to bounce back and forth between bottlenecking on reading
        from the database block device and subsequently the S3 sending
        steps should the processes be at the same stage of the upload
        pipeline: this can have a very negative impact on being able to
        make full use of system resources.

        Furthermore, it desirable to overflowing the page cache: having
        separate tunables for number of simultanious compression jobs
        (which occupy /tmp space and page cache) and number of uploads
        (which affect upload throughput) would help.

        """

        # Get a manifest of files first.
        matches = []

        def raise_walk_error(e):
            raise e

        walker = os.walk(pg_cluster_dir, onerror=raise_walk_error)
        for root, dirnames, filenames in walker:
            is_cluster_toplevel = (os.path.abspath(root) ==
                                   os.path.abspath(pg_cluster_dir))

            # Do not capture any WAL files, although we do want to
            # capture the WAL directory or symlink
            if is_cluster_toplevel:
                if 'pg_xlog' in dirnames:
                    dirnames.remove('pg_xlog')
                    matches.append(os.path.join(root, 'pg_xlog'))

            for filename in filenames:
                if is_cluster_toplevel and filename in ('postmaster.pid',
                                                        'postgresql.conf'):
                    # Do not include the postmaster pid file or the
                    # configuration file in the backup.
                    pass
                else:
                    matches.append(os.path.join(root, filename))

            # Special case for empty directories
            if not filenames:
                matches.append(root)

        backup_s3_prefix = ('{0}/basebackups_{1}/'
                            'base_{file_name}_{file_offset}'
                            .format(self.s3_prefix, FILE_STRUCTURE_VERSION,
                                    **start_backup_info))

        # absolute upload paths are used for telling lzop what to compress
        local_abspaths = [os.path.abspath(match) for match in matches]

        # computed to subtract out extra extraneous absolute path
        # information when storing on S3
        common_local_prefix = os.path.commonprefix(local_abspaths)

        partitions = tar_partition.tar_partitions_plan(
            common_local_prefix, local_abspaths,

            # 1610612736 bytes == 1.5 gigabytes, per partition,
            # non-tunable
            1610612736)

        if rate_limit is None:
            per_process_limit = None
        else:
            per_process_limit = int(rate_limit / pool_size)

        # Reject tiny per-process rate limits.  They should be
        # rejected more nicely elsewhere.
        assert per_process_limit > 0 or per_process_limit is None

        # a list to accumulate async upload jobs
        uploads = []

        total_size = 0

        # Make an attempt to upload extended version metadata
        extended_version_url = backup_s3_prefix + '/extended_version.txt'
        logger.info(
            msg='start upload postgres version metadata',
            detail=('Uploading to {extended_version_url}.'
                    .format(extended_version_url=extended_version_url)))
        s3_worker.uri_put_file(extended_version_url, StringIO(version),
                               content_encoding='text/plain')
        logger.info(msg='postgres version metadata upload complete')

        pool = gevent.pool.Pool(size=pool_size)

        # Enqueue uploads for parallel execution
        try:
            for tpart in partitions:
                total_size += tpart.total_member_size
                uploads.append(pool.apply_async(
                        s3_worker.do_partition_put,
                        [backup_s3_prefix, tpart, per_process_limit,
                         self.gpg_key_id]))
        finally:
            while uploads:
                uploads.pop().get()

            pool.join()

        return backup_s3_prefix, total_size
Example #39
0
 def schedule():
     while True:
         pool.wait_available()
         print('Starting greenlet')
         pool.apply_async(main)
Example #40
0
 def wrapper(*args, **kwds):
     pool = kwds.pop('pool')
     return pool.apply_async(f, args=args, kwds=kwds)
Example #41
0
 def check(self):
     pool = gevent.pool.Pool(len(self.proxies))
     for index, ip in enumerate(self.proxies):
         pool.apply_async(self.ip_delay, (index, ip))
     pool.join()
     self.trigger.emit([0])