def _parse_m3u8(self, vod: typing.Dict) -> typing.Dict:
video_info = self._prepare_video(vod)
def _m3u8(args: typing.Tuple[int, str]) -> typing.Dict:
cum, url = args
variant_m3u8 = m3u8.load(url)
tmp = cum
tar_video = {}
if variant_m3u8.is_variant:
bandwidth_uri = {
p.stream_info.bandwidth: p.uri
for p in variant_m3u8.playlists
}
# best_bandwidth = 2000000 if 2000000 in bandwidth_uri else min(list(bandwidth_uri.keys()))
best_bandwidth = max(list(bandwidth_uri.keys()))
bandwidth_uri = bandwidth_uri[best_bandwidth]
tar_m3u8 = m3u8.load(bandwidth_uri)
for s in tar_m3u8.segments:
tmp += Decimal(str(s.duration)).quantize(
Decimal('0'), rounding=ROUND_HALF_UP)
tar_video[int(tmp)] = s.absolute_uri
return tar_video
group = Group()
result = {}
for tar_video in group.imap_unordered(_m3u8, video_info.items()):
result.update(tar_video)
return result
def main():
group.map(hello_from, xrange(3))
print('Ordered')
ogroup = Group()
for i in ogroup.imap(intensive, xrange(3)):
print(i)
print('Unordered')
igroup = Group()
for i in igroup.imap_unordered(intensive, xrange(3)):
print(i)
def main():
group.map(hello_from, xrange(3))
print('Ordered')
ogroup = Group()
for i in ogroup.imap(intensive, xrange(3)):
print(i)
print('Unordered')
igroup = Group()
for i in igroup.imap_unordered(intensive, xrange(3)):
print(i)
def handle(self, *args, **options):
plogger = logging.getLogger(__name__)
plogger.info("Parallel Update All Feeds")
new_posts_count = 0
start = datetime.now()
feed_urls = Feed.site_objects.all().values_list("url", flat=True)
pool = Group()
for result in pool.imap_unordered(process_feed, feed_urls):
new_posts_count += result
delta = datetime.now() - start
plogger.info("Added %s posts in %d seconds" % (new_posts_count, delta.seconds))
feeds_updated.send(sender=self, instance=self)
def group_parallelism():
group = Group()
def hello_from(n):
print("Size of group %s" % len(group))
print("Hello from Greenlet %s" % id(getcurrent()))
group.map(hello_from, range(3))
def intensive(n):
gevent.sleep(3 - n)
return "task", n
print("Ordered")
ogroup = Group()
for i in ogroup.imap(intensive, range(3)):
print(i)
print("Unordered")
igroup = Group()
for i in igroup.imap_unordered(intensive, range(3)):
print(i)
#!/usr/bin/ env python
# coding:utf-8
__author__ = 'Administrator'
import gevent
from gevent.pool import Group
group = Group()
def group_map(n):
print n
print 'size of group %s' % len(group)
print 'hello from Greenlet %s' % id(gevent.getcurrent())
# 一次性装载3个协程并马上运行
# group.map(group_map, xrange(3))
# 延迟运行，先装载，迭代的时候再真正的执行
# imap_data = group.imap(group_map, xrange(3))
# for i in imap_data:
# print i
# 延迟运行，先装载，迭代的时候再真正的执行，不是顺序执行
imap_data = group.imap_unordered(group_map, xrange(3))
for i in imap_data:
print i
from gevent import getcurrent
from gevent.pool import Group
group = Group()
def hello_from(n):
print "Size of group %s" % len(group)
print "hello from greenlet %s" % id(getcurrent())
group.map(hello_from, xrange(3))
def intensive(n):
gevent.sleep(3 - n)
return "task", n
print "Ordered"
ogroup = Group()
for i in ogroup.imap(intensive, xrange(3)):
print i
print "Unordered"
igroup = Group()
for i in igroup.imap_unordered(intensive, xrange(3)):
print i
# coding: utf-8
# Last modified: 2014 Jun 25 01:31:21 PM
# xh
import gevent
from gevent import getcurrent
from gevent.pool import Group
group = Group()
def hello_from(n):
print('Size of group %s' % len(group))
print('Hello from Greenlet %s' % id(getcurrent()))
group.map(hello_from, xrange(3))
def intensive(n):
gevent.sleep(3 - n)
return 'task', n
print('Ordered')
ogroup = Group()
for i in ogroup.imap(intensive, xrange(3)):
print(i)
print('Unordered')
igroup = Group()
for i in igroup.imap_unordered(intensive, xrange(3)):
print(i)
import gevent
from gevent import getcurrent
from gevent.pool import Group
'''
imap: 返回iterator
map: 返回list, ordered
map_unordered:
imap_unordered
'''
def intensive(n):
gevent.sleep(3-n)
return 'task', n
print('Ordered')
ogroup = Group()
for i in ogroup.imap(intensive, range(3)):
print(i)
print('Unordered')
ugroup = Group()
for i in ugroup.imap_unordered(intensive, range(3)):
print(i)
@author: xiongqiao
@attention：
'''
import gevent
from gevent import getcurrent # 获得当前Greenlet
from gevent.pool import Group
group = Group()
def hello_from(n):
print('Size of group %s' % len(group))
print('Hello from Greenlet %s' % id(getcurrent()))
group.map(hello_from, range(3))
def intensive(n):
gevent.sleep(3 - n)
return 'task', n
print('Ordered') # 有序
ogroup = Group()
for i in ogroup.imap(intensive, range(3)):
print(i)
print('Unordered') # 无序
igroup = Group()
for i in igroup.imap_unordered(intensive, range(5)):
print(i)
def remind(cls, telegramApi: TelegramApiWrapper) -> str:
now = FmtDateTime.now()
hour = now.dateObj.hour
# TODO reset user temp
# Fetch users that aren't blocked and have reminders set
if now.meridies == "AM":
allUserKeys = User.query(User.blocked == False,
User.remindAM == hour,
User.temp == "none").fetch(keys_only=True)
else:
allUserKeys = User.query(User.blocked == False,
User.remindPM == hour,
User.temp == "none").fetch(keys_only=True)
text = STRINGS["window_open"].format(now.time, now.dayOfWeek,
now.shortDate, now.meridies)
SUCCESS = 0
FAILED = 1
BLOCKED = -1
def sendMessage(userKey: ndb.Key):
# gevent somehow messes with the ndb context even though it's the same thread
with cls.ndbClient.context():
# Create message payload
payload = {
"chat_id": str(userKey.id()),
"text": text,
"parse_mode": "HTML",
"reply_markup": TelegramMarkup.TemperatureKeyboard,
}
user: User = userKey.get()
try:
resp = telegramApi.sendMessage(payload)
# User statuses have to be updated right after sending or user may hit an invalid state
# when they report their temperature
if resp["ok"]:
user.temp = User.TEMP_NONE
user.status = UserState.TEMP_REPORT
user.put()
return SUCCESS
else:
if (resp["description"] ==
"Forbidden: bot was blocked by the user"):
user.reset()
user.blocked = True
user.put()
return BLOCKED
else:
return FAILED
logger.error(resp["description"])
except Exception as e:
logger.error(e)
return FAILED
start = time()
pool = Group()
respList = pool.imap_unordered(sendMessage, allUserKeys, maxsize=100)
# Count statuses of reminder
success, failed, blocked = 0, 0, 0
for resp in respList:
if resp == SUCCESS:
success += 1
elif resp == FAILED:
failed += 1
elif resp == BLOCKED:
blocked += 1
elapsedTime = time() - start
rate = len(allUserKeys) / elapsedTime
logStr = f"Reminder sent to {len(allUserKeys)} clients in {elapsedTime:.4f}s ({rate:.2f}/s). Successes: {success}, blocked: {blocked}, failures: {failed}"
logger.info(logStr)
return logStr
def spawn_imap_unordered(self):
igroup = Group()
result = []
with tracer.start_active_span('test'):
for i in igroup.imap_unordered(self.make_http_call, range(3)):
result.append(i)
import gevent
from gevent import getcurrent
from gevent.pool import Group
group = Group()
def hello_from(n):
print('Size of group %s' % len(group))
print('Hello from Greenlet %s' % id(getcurrent()))
group.map(hello_from, range(3))
def intensive(n):
gevent.sleep(3-n)
return 'task',n
print('Orderd')
ogroup = Group()
for i in ogroup.imap(intensive, range(3)): # 相当于itertools.imap 将迭代器迭代每一个元素后作为参数传给func
print(i)
print('Unordered')
igroup = Group()
for i in igroup.imap_unordered(intensive, range(3)): # 返回的迭代结果应以逆序
print(i)
def _do_on_all(self, func):
group = Group()
return group.imap_unordered(func, self.s3.values())
import gevent
from gevent import getcurrent # 获得当前Greenlet
from gevent.pool import Group
group = Group()
def hello_from(n):
print('Size of group %s' % len(group))
print('Hello from Greenlet %s' % id(getcurrent()))
group.map(hello_from, range(3))
def intensive(n):
gevent.sleep(3 - n)
return 'task', n
print('Ordered') # 有序
ogroup = Group()
for i in ogroup.imap(intensive, range(3)):
print(i)
print('Unordered') # 无序
igroup = Group()
for i in igroup.imap_unordered(intensive, range(5)):
print(i)
def sample_unorder():
group = Group()
for i in group.imap_unordered(intensive, xrange(3)):
print i
