def test_apply_async(self): done = Event() def some_work(x): done.set() pool = self.klass(2) pool.apply_async(some_work, ('x', )) done.wait()
def test_reentrant(self): pool = self.klass(1) result = pool.apply(pool.apply, (lambda a: a + 1, (5, ))) self.assertEqual(result, 6) evt = Event() pool.apply_async(evt.set) evt.wait()
def fire_async(self, args=(), kwargs={}, concurrency=None): """Fire the signal asynchronously - each callback in its own greenlet. If concurrency is given, it is the limit on the number of dispatching greenlets to execute concurrently. """ pool = gevent.pool.Pool(size=concurrency) for callback in self.callback.values(): pool.apply_async(callback, args, kwargs)
def _polling_get_status(self, pool): sender = self.target.name logger.debug("POLLING: sender={} signal={}".format( sender, self.signal)) if self.signal.has_receivers_for(sender): logger.debug("POLLING: target={}".format(self.target.nodes)) for cnode in self.target.nodes: pool.apply_async( self.get_status, [[cnode]], callback=lambda x: self.signal.send(sender, message=x)) else: logger.debug("no POLLING: sender={} receiver={}".format( sender, self.signal.receivers))
def jobDispatcher(q): resultsFolder = Configs().get('resultsFolder') xputBuckets = Configs().get('xputBuckets') alpha = Configs().get('alpha') pool = gevent.pool.Pool() while True: args = q.get() pool.apply_async(analyzer, args=( args, resultsFolder, xputBuckets, alpha, ))
def test_reentrant(self): pool = self.klass(1) def reenter(): result = pool.apply(lambda a: a, ('reenter', )) self.assertEqual('reenter', result) pool.apply(reenter) evt = Event() def reenter_async(): pool.apply_async(lambda a: a, ('reenter', )) evt.set() pool.apply_async(reenter_async) evt.wait()
def poi_ori(poi_type): already_merged_city = init_already_merged_city(poi_type=poi_type) conn = base_data_pool.connection() cursor = conn.cursor() cursor.execute('''SELECT id FROM city;''') cids = list(map(lambda x: x[0], cursor.fetchall())) cursor.close() conn.close() for cid in cids: if cid in already_merged_city: continue start = time.time() logger.info('[start][cid: {}]'.format(cid)) pool.apply_async(poi_merge, args=(cid, poi_type)) logger.info('[end][cid: {}][takes: {}]'.format(cid, time.time() - start)) pool.join()
def test_multiple_coros(self): evt = Event() results = [] def producer(): results.append('prod') evt.set() def consumer(): results.append('cons1') evt.wait() results.append('cons2') pool = self.klass(2) done = pool.spawn(consumer) pool.apply_async(producer) done.get() self.assertEquals(['cons1', 'prod', 'cons2'], results)
def download_pic(): conn = pymysql.connect(host='10.10.228.253', user='******', password='******', charset='utf8', db='ServicePlatform') cursor = conn.cursor() cursor.execute("SELECT file_name, source, sid FROM images_attr_daodao_20170929a WHERE `use`=1 LIMIT 100;") start = time.time() _count = 0 for file_name, source, sid in cursor.fetchall(): _count += 1 parent_path = os.path.join(PARENT_PATH, "###".join([source, sid])) if not os.path.exists(parent_path): os.makedirs(parent_path) pool.apply_async(download, ("mioji-attr", file_name, parent_path)) pool.join() cursor.close() conn.close() print("[Total: {}][Takes: {}]".format(_count, time.time() - start))
def test_multiple_coros(self): evt = Event() results = [] def producer(): gevent.sleep(0.001) results.append("prod") evt.set() def consumer(): results.append("cons1") evt.wait() results.append("cons2") pool = self.klass(2) done = pool.spawn(consumer) pool.apply_async(producer) done.get() self.assertEqual(["cons1", "prod", "cons2"], results)
def startBruteforce(sClient, pool, wordlist): for word in open(wordlist, "r"): pool.add( pool.apply_async( isSecretSaslValid, args=(sClient, word.strip(), "sparkSaslUser", True), callback=checkResult, )) pool.join(timeout=30) pool.kill() whine("Could not find the secret", "warn")
def task_run(self): pool = gevent.pool.Pool(2) pool.apply_async(self.task_all, args=(self.worker_list_new, self.webpage_db_new, CHOOSE_DB_NEW)) pool.apply_async(self.task_all, args=(self.worker_list_old, self.webpage_db_old, CHOOSE_DB_OLD)) self.log.info("所有任务加载完成..") pool.join() current_time = 0 sleep_time = 300 self.log.info('完成解析!!, 开始休眠休眠时间为: {rand}'.format(rand=sleep_time)) while is_running and current_time < 300: time.sleep(1) current_time += 1
def _s3_upload_pg_archive(self, archive_filename, pool_size, rate_limit=None): """ Upload archive_filename to s3_url_prefix. """ backup_s3_prefix = ('{0}/basebackups_{1}/base_{2}'.format( self.s3_prefix, FILE_STRUCTURE_VERSION, os.path.basename(archive_filename))) # absolute upload paths are used for telling lzop what to compress local_abspath = os.path.abspath(archive_filename) partitions = tar_partition.archive_partitions_plan( local_abspath, # 1610612736 bytes == 1.5 gigabytes, per partition, # non-tunable 1610612736) if rate_limit is None: per_process_limit = None else: per_process_limit = int(rate_limit / pool_size) # Reject tiny per-process rate limits. They should be # rejected more nicely elsewhere. assert per_process_limit > 0 or per_process_limit is None # a list to accumulate async upload jobs uploads = [] total_size = os.path.getsize(local_abspath) pool = gevent.pool.Pool(size=pool_size) # Enqueue uploads for parallel execution try: for part in partitions: uploads.append( pool.apply_async(s3_worker.do_archive_partition_put, [ backup_s3_prefix, part, per_process_limit, self.gpg_key_id ])) finally: while uploads: uploads.pop().get() pool.join() return backup_s3_prefix, total_size
def _validate_proxy_list(self, proxies, timeout=90): valid_proxies = [] def save_result(p): if p and len(str(p)): valid_proxies.append(p) # 在多进程下面, 只检测部分 if self.process_count > 1: num_per_part = int((len(proxies) + self.process_count - 1) / self.process_count) start = self.process_seq * num_per_part end = (self.process_seq + 1) * num_per_part proxies = proxies[start:end] # 每100 检查一次 total_num = len(proxies) step = 500 pool = gevent.pool.Pool(500) for i in range(0, total_num, step): group = proxies[i:i + step] for proxy in group: pool.apply_async(self._validate_one_proxy, args=(proxy, 'http'), callback=save_result) pool.apply_async(self._validate_one_proxy, args=(proxy, 'https'), callback=save_result) pool.apply_async(self._validate_one_proxy, args=(proxy, 'socks4'), callback=save_result) pool.apply_async(self._validate_one_proxy, args=(proxy, 'socks5'), callback=save_result) is_empty = pool.join(timeout=timeout) if not is_empty: # 还没有检查完整 print("**** validation is not done!") pool.kill() # 即时保存检验过的traffic self.save_proxies(valid_proxies) valid_proxies = [] print("progress: {}/{}".format(i, total_num)) time.sleep(1) return valid_proxies
def pint_test(config): # 创建进程间通信队列 activeq = Queue() notactiveq = Queue() host_list = [] ips = config["ip"].split("-") start_ip = ips[0].split('.') end_ip = ips[1].split('.') base_ip = '.'.join(start_ip[:3]) start_num = start_ip[3] end_num = end_ip[3] for ipnum in range(int(start_num), int(end_num)): host_list.append(f'{base_ip}.{str(ipnum)}') # 创建进程池 pool = Pool(processes=int(config['n'])) for ipaddr in host_list: pool.apply_async(ping_host, args=(activeq, notactiveq, ipaddr)) pool.close() pool.join() # 输出正在使用ip read(activeq) # 输出未被使用ip read(notactiveq)
def _s3_upload_pg_archive(self, archive_filename, pool_size, rate_limit=None): """ Upload archive_filename to s3_url_prefix. """ backup_s3_prefix = ('{0}/basebackups_{1}/base_{2}' .format(self.s3_prefix, FILE_STRUCTURE_VERSION, os.path.basename(archive_filename))) # absolute upload paths are used for telling lzop what to compress local_abspath = os.path.abspath(archive_filename) partitions = tar_partition.archive_partitions_plan(local_abspath, # 1610612736 bytes == 1.5 gigabytes, per partition, # non-tunable 1610612736) if rate_limit is None: per_process_limit = None else: per_process_limit = int(rate_limit / pool_size) # Reject tiny per-process rate limits. They should be # rejected more nicely elsewhere. assert per_process_limit > 0 or per_process_limit is None # a list to accumulate async upload jobs uploads = [] total_size = os.path.getsize(local_abspath) pool = gevent.pool.Pool(size=pool_size) # Enqueue uploads for parallel execution try: for part in partitions: uploads.append(pool.apply_async( s3_worker.do_archive_partition_put, [backup_s3_prefix, part, per_process_limit, self.gpg_key_id])) finally: while uploads: uploads.pop().get() pool.join() return backup_s3_prefix, total_size
def keep_fqsocks_busy(): while True: pool = gevent.pool.Pool(size=16) greenlets = [] for i in range(100): greenlets.append(pool.apply_async(check_twitter_access)) while len(pool) > 0: for greenlet in list(pool): try: greenlet.join(timeout=10) except: pass try: pool.kill() except: pass
def keep_fqsocks_busy(): goagent.GoAgentProxy.GOOGLE_HOSTS = ["goagent-google-ip.fqrouter.com"] goagent.GoAgentProxy.refresh(fqsocks.mandatory_proxies) while True: pool = gevent.pool.Pool(size=16) greenlets = [] for i in range(100): greenlets.append(pool.apply_async(check_baidu_access)) while len(pool) > 0: for greenlet in list(pool): try: greenlet.join(timeout=10) except: pass try: pool.kill() except: pass
def getPingResult(src_ip, count): """ 当前的ping往返平均时间 oid为: 1.3.6.1.2.1.80.1.3.1.6 1.获取交换机所有nqa实例名 2.获取交换机nqa实例对应ping的平均时间(ms) :return: """ # 获取交换机所有nqa实例名,列表中为字典格式name:dst_ip test_name_list = getTestName(src_ip) # 获取交换机nqa实例对应ping的平均时间(ms),测试时间 print "test_name_list: %s" % test_name_list pool = mp.Pool(processes=count) result_tmp = [ pool.apply_async(process_start, args=(src_ip, test_name[0])) for test_name in test_name_list ] ping_result = [p.get() for p in result_tmp] print '*****', ping_result return ping_result
} try: r = requests.get('http://store.nike.com/cn/zh_cn/', proxies=proxies, timeout=(3, 1)) except requests.exceptions.ConnectionError: return except requests.exceptions.ReadTimeout: return end = time.time() delay = '{:.0f}ms'.format((end-start)*1000) queue.put([index, delay]) if __name__ == '__main__': with open('give.txt', 'r') as f: ips = f.read().strip().split('\n') pool = gevent.pool.Pool(len(ips)) queue = gevent.queue.Queue() for index, ip in enumerate(ips): pool.apply_async(ip_delay, (index, ip)) pool.join() # ip_delay(00, None) nums = [] while True: if queue.qsize() > 0: task = queue.get() print(task) nums.append(task[0]) else: break nums.sort() print(nums)
def _s3_upload_pg_cluster_dir(self, start_backup_info, pg_cluster_dir, version, pool_size, rate_limit=None): """ Upload to s3_url_prefix from pg_cluster_dir This function ignores the directory pg_xlog, which contains WAL files and are not generally part of a base backup. Note that this is also lzo compresses the files: thus, the number of pooled processes involves doing a full sequential scan of the uncompressed Postgres heap file that is pipelined into lzo. Once lzo is completely finished (necessary to have access to the file size) the file is sent to S3. TODO: Investigate an optimization to decouple the compression and upload steps to make sure that the most efficient possible use of pipelining of network and disk resources occurs. Right now it possible to bounce back and forth between bottlenecking on reading from the database block device and subsequently the S3 sending steps should the processes be at the same stage of the upload pipeline: this can have a very negative impact on being able to make full use of system resources. Furthermore, it desirable to overflowing the page cache: having separate tunables for number of simultanious compression jobs (which occupy /tmp space and page cache) and number of uploads (which affect upload throughput) would help. """ parts = tar_partition.partition(pg_cluster_dir) backup_s3_prefix = ('{0}/basebackups_{1}/' 'base_{file_name}_{file_offset}' .format(self.s3_prefix, FILE_STRUCTURE_VERSION, **start_backup_info)) if rate_limit is None: per_process_limit = None else: per_process_limit = int(rate_limit / pool_size) # Reject tiny per-process rate limits. They should be # rejected more nicely elsewhere. assert per_process_limit > 0 or per_process_limit is None # a list to accumulate async upload jobs uploads = [] total_size = 0 # Make an attempt to upload extended version metadata extended_version_url = backup_s3_prefix + '/extended_version.txt' logger.info( msg='start upload postgres version metadata', detail=('Uploading to {extended_version_url}.' .format(extended_version_url=extended_version_url))) s3_worker.uri_put_file(extended_version_url, StringIO(version), content_encoding='text/plain') logger.info(msg='postgres version metadata upload complete') pool = gevent.pool.Pool(size=pool_size) # Enqueue uploads for parallel execution try: for tpart in parts: total_size += tpart.total_member_size uploads.append(pool.apply_async( s3_worker.do_partition_put, [backup_s3_prefix, tpart, per_process_limit, self.gpg_key_id])) finally: while uploads: uploads.pop().get() pool.join() return backup_s3_prefix, total_size
def wrapper(*args, **kwds): pool = kwds.pop('pool') return pool.apply_async(f, args=args, kwds=kwds)
def reenter_async(): pool.apply_async(lambda a: a, ('reenter', )) evt.set()
# @Time : 2017/9/5 下午4:42 # @Author : Hou Rong # @Site : # @File : test.py # @Software: PyCharm from gevent.monkey import patch_all patch_all() import gevent.pool pool = gevent.pool.Pool() from rate_limit_redis import RateLimitRedis import redis import time redis_conn = redis.Redis() key = 'hourong.me' @RateLimitRedis(redis_conn=redis_conn, key=key, period=2) def do_something(args): print("start", args, time.time()) time.sleep(10) print("end", args, time.time()) for i in range(5): pool.apply_async(do_something, (i, )) pool.join()
def _polling_get_cell_status(self, pool): sender = self.target.name if self.signal.has_receivers_for(sender): pool.apply_async( self.get_cell_status, callback=lambda x: self.signal.send(sender, message=x))
def fetch(self): pool = gevent.pool.Pool(size=6) for repo in self._repository_iter(): pool.apply_async(repo.fetch, [self.dry_run]) pool.join()
def _s3_upload_pg_cluster_dir(self, start_backup_info, pg_cluster_dir, version, pool_size, rate_limit=None): """ Upload to s3_url_prefix from pg_cluster_dir This function ignores the directory pg_xlog, which contains WAL files and are not generally part of a base backup. Note that this is also lzo compresses the files: thus, the number of pooled processes involves doing a full sequential scan of the uncompressed Postgres heap file that is pipelined into lzo. Once lzo is completely finished (necessary to have access to the file size) the file is sent to S3. TODO: Investigate an optimization to decouple the compression and upload steps to make sure that the most efficient possible use of pipelining of network and disk resources occurs. Right now it possible to bounce back and forth between bottlenecking on reading from the database block device and subsequently the S3 sending steps should the processes be at the same stage of the upload pipeline: this can have a very negative impact on being able to make full use of system resources. Furthermore, it desirable to overflowing the page cache: having separate tunables for number of simultanious compression jobs (which occupy /tmp space and page cache) and number of uploads (which affect upload throughput) would help. """ # Get a manifest of files first. matches = [] def raise_walk_error(e): raise e walker = os.walk(pg_cluster_dir, onerror=raise_walk_error) for root, dirnames, filenames in walker: is_cluster_toplevel = (os.path.abspath(root) == os.path.abspath(pg_cluster_dir)) # Do not capture any WAL files, although we do want to # capture the WAL directory or symlink if is_cluster_toplevel: if 'pg_xlog' in dirnames: dirnames.remove('pg_xlog') matches.append(os.path.join(root, 'pg_xlog')) for filename in filenames: if is_cluster_toplevel and filename in ('postmaster.pid', 'postgresql.conf'): # Do not include the postmaster pid file or the # configuration file in the backup. pass else: matches.append(os.path.join(root, filename)) # Special case for empty directories if not filenames: matches.append(root) backup_s3_prefix = ('{0}/basebackups_{1}/' 'base_{file_name}_{file_offset}' .format(self.s3_prefix, FILE_STRUCTURE_VERSION, **start_backup_info)) # absolute upload paths are used for telling lzop what to compress local_abspaths = [os.path.abspath(match) for match in matches] # computed to subtract out extra extraneous absolute path # information when storing on S3 common_local_prefix = os.path.commonprefix(local_abspaths) partitions = tar_partition.tar_partitions_plan( common_local_prefix, local_abspaths, # 1610612736 bytes == 1.5 gigabytes, per partition, # non-tunable 1610612736) if rate_limit is None: per_process_limit = None else: per_process_limit = int(rate_limit / pool_size) # Reject tiny per-process rate limits. They should be # rejected more nicely elsewhere. assert per_process_limit > 0 or per_process_limit is None # a list to accumulate async upload jobs uploads = [] total_size = 0 # Make an attempt to upload extended version metadata extended_version_url = backup_s3_prefix + '/extended_version.txt' logger.info(msg='start upload postgres version metadata', detail=('Uploading to {extended_version_url}.' .format(extended_version_url=extended_version_url))) s3_worker.uri_put_file(extended_version_url, StringIO(version), content_encoding='text/plain') logger.info(msg='postgres version metadata upload complete') pool = gevent.pool.Pool(size=pool_size) # Enqueue uploads for parallel execution try: for tpart in partitions: total_size += tpart.total_member_size uploads.append(pool.apply_async( s3_worker.do_partition_put, [backup_s3_prefix, tpart, per_process_limit, self.gpg_key_id])) finally: while uploads: uploads.pop().get() pool.join() return backup_s3_prefix, total_size
def post(self): pool = self.application.settings.get('POSTpool') args = self.request.arguments LOG_ACTION(logger, 'POST:' + str(args)) pool.apply_async(postHandler, (args, ), callback=self._callback)
def _download_res( self, filepath, rate, uploading=True, callback=None, cb_kwargs=None): try: peers = self.get_peers() self._record_get_peer_ts() peers_num = len(peers) count = 0 # just get resource size while True: ip, port = peers[count] logger.info('get resource size') try: ret = self._requests_session.get( "{protocol}{ip}:{port}/?{res}" .format( protocol=PROTOCOL, ip=ip, port=port, res=urlencode({'res_url': self.upload_res_url})), stream=True, headers={"Range": "bytes=0-0"}, timeout=1) if ret.ok: #: bytes=0-1/17) content_range = ret.headers.get("Content-Range") res_length = content_range.split('/')[-1] break else: logger.warn( 'get piece from ip: %s port: %s error, code: %s ' % (ip, port, ret.status_code)) count += 1 self.del_from_tracker(ip=ip, peer_port=port) except ConnectionError: logger.warn( 'get piece from ip: %s port: %s error ConnectionError' % (ip, port)) count += 1 self.del_from_tracker(ip=ip, peer_port=port) except Timeout: logger.warn( 'get piece from ip: %s port: %s error Timeout' % (ip, port)) count += 1 self.del_from_tracker(ip=ip, peer_port=port) finally: if count >= peers_num: logger.warn("No peers avaliable") peers = self.get_peers() peers_num = len(peers) count = 0 logger.info('%s is size of %s' % (self.upload_res_url, sizeof_fmt_human(res_length))) self.piece_file = PieceFile(res_length, filepath) pool_work_num = 15 pool_q_size = pool_work_num * 2 pool = gevent.pool.Pool(pool_work_num) self.start_ready_upload_thread() if rate: self.download_rate = rate else: rate = self.download_rate if rate: self.token_bucket = TokenBucket(rate) while self.piece_file.has_unalloc(): args_list = list() for peer in peers: if peer not in self._peer_in_conn: args_list.append((peer, None)) [pool.apply_async(self._download_piece_thread, *args) for args in args_list[:pool_q_size]] # update peers if peer run out while pool.full(): gevent.sleep(0.2) if not self.piece_file.has_empty(): pool.join() logger.debug( 'test get_empty_block: %s' % self.piece_file.get_empty_piece()) logger.debug('peer in connection: %s' % self._peer_in_conn) if self.piece_file.has_unalloc(): try: tv = self._get_last_get_peer_tv() if tv < GET_PEER_INTERVAL: gevent.sleep(GET_PEER_INTERVAL - tv) g = gevent.spawn(self.get_peers) peers = g.get() self._record_get_peer_ts() except NoPeersFound: # if pool.workRequests: if pool_work_num - pool.free_count() > 0: # some remained piece maybe on the way pool.join() if self.piece_file.has_unalloc(): tv = self._get_last_get_peer_tv() if tv > GET_PEER_INTERVAL: gevent.sleep(GET_PEER_INTERVAL - tv) g = gevent.spawn(self.get_peers) peers = g.get() self._record_get_peer_ts() else: break else: logger.error("no worker running, and get no peers") raise else: break logger.info('File is complete, size: %s' % self.piece_file.get_real_filesize()) except NoPeersFound: if self.fallback: logger.info('Use fallback way to get resouce') try: res_length = get_res_length(self.res_url) except ConnectionError: raise OriginURLConnectError(self.res_url) logger.info( 'get resource length %s' % sizeof_fmt_human(res_length)) if not self.piece_file: self.piece_file = PieceFile(res_length, filepath) self.start_ready_upload_thread() http_download_to_piecefile( self.res_url, self.piece_file) else: self.deactivate() raise # self.piece_file.tofile() self.res_is_downloaded = True if callback: logger.info('Run callback') callback(**cb_kwargs)
#!/usr/bin/env python # -*- coding: utf-8 -*- # @Time : 2017/10/31 上午9:39 # @Author : Hou Rong # @Site : # @File : gevent_pool_timout_test.py # @Software: PyCharm import gevent.monkey gevent.monkey.patch_all() import gevent.pool import time import random pool = gevent.pool.Pool(size=10) def hello_world(): t = random.randint(1, 10) time.sleep(t) print("Hello World, {}".format(t)) if __name__ == '__main__': for i in range(10): pool.apply_async(hello_world, ()) pool.join(timeout=5)
def _s3_upload_pg_cluster_dir(self, start_backup_info, pg_cluster_dir, version, pool_size, rate_limit=None): """ Upload to s3_url_prefix from pg_cluster_dir This function ignores the directory pg_xlog, which contains WAL files and are not generally part of a base backup. Note that this is also lzo compresses the files: thus, the number of pooled processes involves doing a full sequential scan of the uncompressed Postgres heap file that is pipelined into lzo. Once lzo is completely finished (necessary to have access to the file size) the file is sent to S3. TODO: Investigate an optimization to decouple the compression and upload steps to make sure that the most efficient possible use of pipelining of network and disk resources occurs. Right now it possible to bounce back and forth between bottlenecking on reading from the database block device and subsequently the S3 sending steps should the processes be at the same stage of the upload pipeline: this can have a very negative impact on being able to make full use of system resources. Furthermore, it desirable to overflowing the page cache: having separate tunables for number of simultanious compression jobs (which occupy /tmp space and page cache) and number of uploads (which affect upload throughput) would help. """ # Get a manifest of files first. matches = [] def raise_walk_error(e): raise e walker = os.walk(pg_cluster_dir, onerror=raise_walk_error) for root, dirnames, filenames in walker: is_cluster_toplevel = (os.path.abspath(root) == os.path.abspath(pg_cluster_dir)) # Do not capture any WAL files, although we do want to # capture the WAL directory or symlink if is_cluster_toplevel: if 'pg_xlog' in dirnames: dirnames.remove('pg_xlog') matches.append(os.path.join(root, 'pg_xlog')) for filename in filenames: if is_cluster_toplevel and filename in ('postmaster.pid', 'postgresql.conf'): # Do not include the postmaster pid file or the # configuration file in the backup. pass else: matches.append(os.path.join(root, filename)) # Special case for empty directories if not filenames: matches.append(root) backup_s3_prefix = ('{0}/basebackups_{1}/' 'base_{file_name}_{file_offset}' .format(self.s3_prefix, FILE_STRUCTURE_VERSION, **start_backup_info)) # absolute upload paths are used for telling lzop what to compress local_abspaths = [os.path.abspath(match) for match in matches] # computed to subtract out extra extraneous absolute path # information when storing on S3 common_local_prefix = os.path.commonprefix(local_abspaths) partitions = tar_partition.tar_partitions_plan( common_local_prefix, local_abspaths, # 1610612736 bytes == 1.5 gigabytes, per partition, # non-tunable 1610612736) if rate_limit is None: per_process_limit = None else: per_process_limit = int(rate_limit / pool_size) # Reject tiny per-process rate limits. They should be # rejected more nicely elsewhere. assert per_process_limit > 0 or per_process_limit is None # a list to accumulate async upload jobs uploads = [] total_size = 0 # Make an attempt to upload extended version metadata extended_version_url = backup_s3_prefix + '/extended_version.txt' logger.info( msg='start upload postgres version metadata', detail=('Uploading to {extended_version_url}.' .format(extended_version_url=extended_version_url))) s3_worker.uri_put_file(extended_version_url, StringIO(version), content_encoding='text/plain') logger.info(msg='postgres version metadata upload complete') pool = gevent.pool.Pool(size=pool_size) # Enqueue uploads for parallel execution try: for tpart in partitions: total_size += tpart.total_member_size uploads.append(pool.apply_async( s3_worker.do_partition_put, [backup_s3_prefix, tpart, per_process_limit, self.gpg_key_id])) finally: while uploads: uploads.pop().get() pool.join() return backup_s3_prefix, total_size
def schedule(): while True: pool.wait_available() print('Starting greenlet') pool.apply_async(main)
def check(self): pool = gevent.pool.Pool(len(self.proxies)) for index, ip in enumerate(self.proxies): pool.apply_async(self.ip_delay, (index, ip)) pool.join() self.trigger.emit([0])