def _parse_m3u8(self, vod: typing.Dict) -> typing.Dict: video_info = self._prepare_video(vod) def _m3u8(args: typing.Tuple[int, str]) -> typing.Dict: cum, url = args variant_m3u8 = m3u8.load(url) tmp = cum tar_video = {} if variant_m3u8.is_variant: bandwidth_uri = { p.stream_info.bandwidth: p.uri for p in variant_m3u8.playlists } # best_bandwidth = 2000000 if 2000000 in bandwidth_uri else min(list(bandwidth_uri.keys())) best_bandwidth = max(list(bandwidth_uri.keys())) bandwidth_uri = bandwidth_uri[best_bandwidth] tar_m3u8 = m3u8.load(bandwidth_uri) for s in tar_m3u8.segments: tmp += Decimal(str(s.duration)).quantize( Decimal('0'), rounding=ROUND_HALF_UP) tar_video[int(tmp)] = s.absolute_uri return tar_video group = Group() result = {} for tar_video in group.imap_unordered(_m3u8, video_info.items()): result.update(tar_video) return result
def main(): group.map(hello_from, xrange(3)) print('Ordered') ogroup = Group() for i in ogroup.imap(intensive, xrange(3)): print(i) print('Unordered') igroup = Group() for i in igroup.imap_unordered(intensive, xrange(3)): print(i)
def handle(self, *args, **options): plogger = logging.getLogger(__name__) plogger.info("Parallel Update All Feeds") new_posts_count = 0 start = datetime.now() feed_urls = Feed.site_objects.all().values_list("url", flat=True) pool = Group() for result in pool.imap_unordered(process_feed, feed_urls): new_posts_count += result delta = datetime.now() - start plogger.info("Added %s posts in %d seconds" % (new_posts_count, delta.seconds)) feeds_updated.send(sender=self, instance=self)
def group_parallelism(): group = Group() def hello_from(n): print("Size of group %s" % len(group)) print("Hello from Greenlet %s" % id(getcurrent())) group.map(hello_from, range(3)) def intensive(n): gevent.sleep(3 - n) return "task", n print("Ordered") ogroup = Group() for i in ogroup.imap(intensive, range(3)): print(i) print("Unordered") igroup = Group() for i in igroup.imap_unordered(intensive, range(3)): print(i)
#!/usr/bin/ env python # coding:utf-8 __author__ = 'Administrator' import gevent from gevent.pool import Group group = Group() def group_map(n): print n print 'size of group %s' % len(group) print 'hello from Greenlet %s' % id(gevent.getcurrent()) # 一次性装载3个协程并马上运行 # group.map(group_map, xrange(3)) # 延迟运行,先装载,迭代的时候再真正的执行 # imap_data = group.imap(group_map, xrange(3)) # for i in imap_data: # print i # 延迟运行,先装载,迭代的时候再真正的执行,不是顺序执行 imap_data = group.imap_unordered(group_map, xrange(3)) for i in imap_data: print i
from gevent import getcurrent from gevent.pool import Group group = Group() def hello_from(n): print "Size of group %s" % len(group) print "hello from greenlet %s" % id(getcurrent()) group.map(hello_from, xrange(3)) def intensive(n): gevent.sleep(3 - n) return "task", n print "Ordered" ogroup = Group() for i in ogroup.imap(intensive, xrange(3)): print i print "Unordered" igroup = Group() for i in igroup.imap_unordered(intensive, xrange(3)): print i
# coding: utf-8 # Last modified: 2014 Jun 25 01:31:21 PM # xh import gevent from gevent import getcurrent from gevent.pool import Group group = Group() def hello_from(n): print('Size of group %s' % len(group)) print('Hello from Greenlet %s' % id(getcurrent())) group.map(hello_from, xrange(3)) def intensive(n): gevent.sleep(3 - n) return 'task', n print('Ordered') ogroup = Group() for i in ogroup.imap(intensive, xrange(3)): print(i) print('Unordered') igroup = Group() for i in igroup.imap_unordered(intensive, xrange(3)): print(i)
import gevent from gevent import getcurrent from gevent.pool import Group ''' imap: 返回iterator map: 返回list, ordered map_unordered: imap_unordered ''' def intensive(n): gevent.sleep(3-n) return 'task', n print('Ordered') ogroup = Group() for i in ogroup.imap(intensive, range(3)): print(i) print('Unordered') ugroup = Group() for i in ugroup.imap_unordered(intensive, range(3)): print(i)
@author: xiongqiao @attention: ''' import gevent from gevent import getcurrent # 获得当前Greenlet from gevent.pool import Group group = Group() def hello_from(n): print('Size of group %s' % len(group)) print('Hello from Greenlet %s' % id(getcurrent())) group.map(hello_from, range(3)) def intensive(n): gevent.sleep(3 - n) return 'task', n print('Ordered') # 有序 ogroup = Group() for i in ogroup.imap(intensive, range(3)): print(i) print('Unordered') # 无序 igroup = Group() for i in igroup.imap_unordered(intensive, range(5)): print(i)
def remind(cls, telegramApi: TelegramApiWrapper) -> str: now = FmtDateTime.now() hour = now.dateObj.hour # TODO reset user temp # Fetch users that aren't blocked and have reminders set if now.meridies == "AM": allUserKeys = User.query(User.blocked == False, User.remindAM == hour, User.temp == "none").fetch(keys_only=True) else: allUserKeys = User.query(User.blocked == False, User.remindPM == hour, User.temp == "none").fetch(keys_only=True) text = STRINGS["window_open"].format(now.time, now.dayOfWeek, now.shortDate, now.meridies) SUCCESS = 0 FAILED = 1 BLOCKED = -1 def sendMessage(userKey: ndb.Key): # gevent somehow messes with the ndb context even though it's the same thread with cls.ndbClient.context(): # Create message payload payload = { "chat_id": str(userKey.id()), "text": text, "parse_mode": "HTML", "reply_markup": TelegramMarkup.TemperatureKeyboard, } user: User = userKey.get() try: resp = telegramApi.sendMessage(payload) # User statuses have to be updated right after sending or user may hit an invalid state # when they report their temperature if resp["ok"]: user.temp = User.TEMP_NONE user.status = UserState.TEMP_REPORT user.put() return SUCCESS else: if (resp["description"] == "Forbidden: bot was blocked by the user"): user.reset() user.blocked = True user.put() return BLOCKED else: return FAILED logger.error(resp["description"]) except Exception as e: logger.error(e) return FAILED start = time() pool = Group() respList = pool.imap_unordered(sendMessage, allUserKeys, maxsize=100) # Count statuses of reminder success, failed, blocked = 0, 0, 0 for resp in respList: if resp == SUCCESS: success += 1 elif resp == FAILED: failed += 1 elif resp == BLOCKED: blocked += 1 elapsedTime = time() - start rate = len(allUserKeys) / elapsedTime logStr = f"Reminder sent to {len(allUserKeys)} clients in {elapsedTime:.4f}s ({rate:.2f}/s). Successes: {success}, blocked: {blocked}, failures: {failed}" logger.info(logStr) return logStr
def spawn_imap_unordered(self): igroup = Group() result = [] with tracer.start_active_span('test'): for i in igroup.imap_unordered(self.make_http_call, range(3)): result.append(i)
import gevent from gevent import getcurrent from gevent.pool import Group group = Group() def hello_from(n): print('Size of group %s' % len(group)) print('Hello from Greenlet %s' % id(getcurrent())) group.map(hello_from, range(3)) def intensive(n): gevent.sleep(3-n) return 'task',n print('Orderd') ogroup = Group() for i in ogroup.imap(intensive, range(3)): # 相当于itertools.imap 将迭代器迭代每一个元素后作为参数传给func print(i) print('Unordered') igroup = Group() for i in igroup.imap_unordered(intensive, range(3)): # 返回的迭代结果应以逆序 print(i)
def _do_on_all(self, func): group = Group() return group.imap_unordered(func, self.s3.values())
import gevent from gevent import getcurrent # 获得当前Greenlet from gevent.pool import Group group = Group() def hello_from(n): print('Size of group %s' % len(group)) print('Hello from Greenlet %s' % id(getcurrent())) group.map(hello_from, range(3)) def intensive(n): gevent.sleep(3 - n) return 'task', n print('Ordered') # 有序 ogroup = Group() for i in ogroup.imap(intensive, range(3)): print(i) print('Unordered') # 无序 igroup = Group() for i in igroup.imap_unordered(intensive, range(5)): print(i)
def sample_unorder(): group = Group() for i in group.imap_unordered(intensive, xrange(3)): print i