def test_multi_exceptions(self): with ExpectLog(app_log, "Multiple exceptions in yield list"): with self.assertRaises(RuntimeError) as cm: yield gen.Multi( [ self.async_exception(RuntimeError("error 1")), self.async_exception(RuntimeError("error 2")), ] ) self.assertEqual(str(cm.exception), "error 1") # With only one exception, no error is logged. with self.assertRaises(RuntimeError): yield gen.Multi( [self.async_exception(RuntimeError("error 1")), self.async_future(2)] ) # Exception logging may be explicitly quieted. with self.assertRaises(RuntimeError): yield gen.Multi( [ self.async_exception(RuntimeError("error 1")), self.async_exception(RuntimeError("error 2")), ], quiet_exceptions=RuntimeError, )
def overload(n=10): usernames = ['testuser%s' % i for i in range(n)] while True: tasks = [ gen.Task(db.do_request, 'PutItem', { "Item": { "username": { "S": username } }, "TableName": "users_dev" }) for username in usernames ] resp = yield gen.Multi(tasks) logging.info('puts resp %s ' % [r.code for r in resp]) tasks = [ gen.Task(db.get_item, 'users_dev', username) for username in usernames ] resp = yield gen.Multi(tasks) logging.info('gets resp %s ' % [r.code for r in resp])
def test_login(): username = sys.argv[1] password = sys.argv[2] # check result.. #torrent = 'http://www.clearbits.net/get/503-control-alt-deus---made-of-fire.torrent' hash = ''.join([ random.choice(list('abcdef') + map(str, range(10))) for _ in range(40) ]) torrent = 'magnet:?xt=urn:btih:%s' % hash for _ in range(1): client = Client(username, password) client.sync() yield gen.Task(asyncsleep, 1) #client.add_url(torrent) client.stop() tasks = [] for hash, torrent in client.torrents.items(): if torrent.get('progress') == 1000: tasks.append(gen.Task(torrent.fetch_files)) tasks.append(gen.Task(torrent.fetch_metadata)) responses = yield gen.Multi(tasks) logging.info('responses %s' % [r.code for r in responses]) tasks = [] for hash, torrent in client.torrents.items(): if torrent.get('progress') == 1000: for file in torrent.files: link = file.webseed_link() print link request = tornado.httpclient.HTTPRequest(link, validate_cert=False) tasks.append(gen.Task(httpclient.fetch, request)) while tasks: some_tasks = [tasks.pop() for _ in range(5)] logging.info('executing tasks of len %s' % len(some_tasks)) responses = yield gen.Multi(some_tasks) logging.info('responses %s' % [(r.code, len(r.body)) for r in responses]) if False: tasks = [] for hash, torrent in client.torrents.items(): if torrent.get('progress') == 1000: link = torrent.webseed_link() print torrent.get('name'), torrent.get('progress'), link request = tornado.httpclient.HTTPRequest(link, validate_cert=False) tasks.append(gen.Task(httpclient.fetch, request)) responses = yield gen.Multi(tasks) logging.info('responses %s' % [r.code for r in responses])
def mul_process(): executor = concurrent.futures.ProcessPoolExecutor( max_workers=os.cpu_count()) f_list = [executor.submit(is_prime_i, i) for i in PRIMES] res_list = yield gen.Multi(f_list) print(res_list) executor.shutdown()
def f(): # callbacks run at different times responses = yield gen.Multi([ gen.Task(self.delay_callback, 3, arg="v1"), gen.Task(self.delay_callback, 1, arg="v2"), ]) self.assertEqual(responses, ["v1", "v2"]) self.stop()
def f(): # callbacks run at different times responses = yield gen.Multi(dict( foo=gen.Task(self.delay_callback, 3, arg="v1"), bar=gen.Task(self.delay_callback, 1, arg="v2"), )) self.assertEqual(responses, dict(foo="v1", bar="v2")) self.stop()
def organiza_pedidos_e_envia_cozinha(self, pedidos): """Esta função esta responsavel por organizar os pedidos e enviar à cozinha. Ela gera seu JSON e realiza o pedido para a cozinha. Retorna a lista com as tasks de pedidos que estão prontos. """ # Criar lista de pedidos a serem preparados pedidos_sendo_preparados = [] # Vai em cada um dos pedidos for pedido in pedidos: # Gera um json para se comunicar com a cozinha pedido_json = json.dumps({"pedido": pedido}) # Realiza o pedido à cozinha pedido_preparando = self.envia_pedido_cozinha(pedido_json) # Adicionando o pedido a liista de pedidos à serem preparados pedidos_sendo_preparados.append(pedido_preparando) # Retorna uma lista de tasks dentro de uma Multi # que sera cuidada pelo IOLoop return gen.Multi(pedidos_sendo_preparados)
def create_domains(domains): yield gen.Multi([gen.Task(db.create_domain, domain) for domain in domains])
def _NotifyFollowers(cls, client, viewpoint_id, followers, invalidate, activity_func, inc_badge=False, always_notify=False): """Adds a notification for each of the given followers that the specified viewpoint has structurally changed. If "invalidate" is a dict, then uses that directly. Otherwise, assumes it's a function that takes a follower id and returns the invalidate dict for that follower. If "always_notify" is true, then always send notifications, even to removed followers. In order to minimize undesirable client artifacts caused by reads of half-committed data, we will commit updates in this order: 1. In parallel: a. Create the activity. b. Update all Followed records (of all followers in the viewpoint). 2. In parallel: a. Update update_seq in the viewpoint. b. Update viewed_seq in the sending follower. 3. In parallel: a. For each follower: i. Create notification ii. Send alert """ from viewfinder.backend.db.viewpoint import Viewpoint from viewfinder.backend.op.alert_manager import AlertManager # Get the current operation, which provides the calling user and the op timestamp. operation = NotificationManager._GetOperation() @gen.coroutine def _NotifyOneFollower(viewpoint, seq_num_pair, activity, follower, follower_settings): """Creates a notification for the follower and sends an alert if configured to do so.""" # If follower has been removed, do not send notifications or alerts to it. if follower.IsRemoved() and not always_notify: return # Get the invalidate dict. if invalidate is None or isinstance(invalidate, dict): foll_invalidate = invalidate else: foll_invalidate = invalidate(follower.user_id) # Don't send alert or increment badge for the user that is creating the activity and # sending the NotificationManager. is_sending_user = follower.user_id == operation.user_id # Create the notification for the follower. # Update the Followed index, which orders viewpoints by timestamp of last update. notification = yield Notification.CreateForUser( client, operation, follower.user_id, activity.name, invalidate=foll_invalidate, activity_id=activity.activity_id, viewpoint_id=viewpoint_id, seq_num_pair=seq_num_pair, inc_badge=inc_badge and not is_sending_user) if not is_sending_user: yield AlertManager.SendFollowerAlert(client, follower.user_id, notification.badge, viewpoint, follower, follower_settings, activity) # We want a locked viewpoint while updating its sequence numbers and the corresponding Followed rows. # Locking also prevents race conditions where new followers are added during iteration. Viewpoint.AssertViewpointLockAcquired(viewpoint_id) # Get affected viewpoint and the follower sending the notification, if it's available. viewpoint = yield gen.Task(Viewpoint.Query, client, viewpoint_id, None) sending_follower = next((follower for follower in followers if follower.user_id == operation.user_id), None) # Update the viewpoint and follower sequence numbers, but do not commit until after Followed # records are updated (since we're updating the viewpoint's "last_updated" at that time anyway). viewpoint.update_seq += 1 if sending_follower is not None: sending_follower.viewed_seq += 1 seq_num_pair = (viewpoint.update_seq, sending_follower.viewed_seq) else: seq_num_pair = (viewpoint.update_seq, None) # Create the activity. activity_task = activity_func(client, operation.user_id, viewpoint_id, viewpoint.update_seq) # Get account settings for each follower in order to determine what level of alerts they'd like. follower_keys = [ AccountSettings.ConstructKey(follower.user_id) for follower in followers ] settings_task = gen.Task(AccountSettings.BatchQuery, client, follower_keys, None, must_exist=False) # Update all Followed records. followed_task = gen.Multi([ gen.Task(Followed.UpdateDateUpdated, client, follower.user_id, viewpoint_id, viewpoint.last_updated, operation.timestamp) for follower in followers ]) activity, all_follower_settings, _ = yield [ activity_task, settings_task, followed_task ] # Now that the Followed records have been updated, update the viewpoint's "last_updated" attribute. # This must be done afterwards so that the previous value of last_updated is known, even if the # operation fails and restarts. if operation.timestamp > viewpoint.last_updated: viewpoint.last_updated = operation.timestamp # Commit changes to update_seq and the sending follower's viewed_seq. yield [ gen.Task(viewpoint.Update, client), gen.Task(sending_follower.Update, client) if sending_follower is not None else util.GenConstant(None) ] # Visit each follower and generate notifications and alerts for it. yield [ _NotifyOneFollower(viewpoint, seq_num_pair, activity, follower, follower_settings) for follower, follower_settings in zip(followers, all_follower_settings) ]
def _QueryEpisodesForArchive(client, obj_store, user_id, episode_ids): """Queries posts from the specified episodes. """ def _MakePhotoDict(post, photo, user_post, user_photo): ph_dict = photo.MakeMetadataDict(post, user_post, user_photo) # Do not return access URLs for posts which have been removed. if not post.IsRemoved(): ph_dict['full_get_url'] = photo_store.GeneratePhotoUrl( obj_store, ph_dict['photo_id'], '.f') return ph_dict # Get all requested episodes, along with posts for each episode. episode_keys = [db_client.DBKey(ep_id, None) for ep_id in episode_ids] post_tasks = [] for ep_id in episode_ids: post_tasks.append( gen.Task(Post.RangeQuery, client, ep_id, None, None, None, excl_start_key=None)) episodes, posts_list = yield [ gen.Task(Episode.BatchQuery, client, episode_keys, None, must_exist=False), gen.Multi(post_tasks) ] # Get viewpoint records for all viewpoints containing episodes. viewpoint_keys = [ db_client.DBKey(viewpoint_id, None) for viewpoint_id in set( ep.viewpoint_id for ep in episodes if ep is not None) ] # Get follower records for all viewpoints containing episodes, along with photo and user post objects. follower_keys = [ db_client.DBKey(user_id, db_key.hash_key) for db_key in viewpoint_keys ] all_posts = [ post for posts in posts_list if posts is not None for post in posts ] photo_keys = [db_client.DBKey(post.photo_id, None) for post in all_posts] user_post_keys = [ db_client.DBKey(user_id, Post.ConstructPostId(post.episode_id, post.photo_id)) for post in all_posts ] if user_id: # TODO(ben): we can probably skip this for the web view user_photo_task = gen.Task( UserPhoto.BatchQuery, client, [db_client.DBKey(user_id, post.photo_id) for post in all_posts], None, must_exist=False) else: user_photo_task = util.GenConstant(None) viewpoints, followers, photos, user_posts, user_photos = yield [ gen.Task(Viewpoint.BatchQuery, client, viewpoint_keys, None, must_exist=False), gen.Task(Follower.BatchQuery, client, follower_keys, None, must_exist=False), gen.Task(Photo.BatchQuery, client, photo_keys, None), gen.Task(UserPost.BatchQuery, client, user_post_keys, None, must_exist=False), user_photo_task, ] # Get set of viewpoint ids to which the current user has access. viewable_viewpoint_ids = set( viewpoint.viewpoint_id for viewpoint, follower in zip(viewpoints, followers) if _CanViewViewpointContent(viewpoint, follower)) response_dict = {'episodes': []} for ep_id, episode, posts in zip(episode_ids, episodes, posts_list): # Gather list of (post, photo, user_post) tuples for this episode. photo_info_list = [] for post in posts: photo = photos.pop(0) user_post = user_posts.pop(0) user_photo = user_photos.pop( 0) if user_photos is not None else None assert photo.photo_id == post.photo_id, (episode, post, photo) if user_photo: assert user_photo.photo_id == photo.photo_id assert user_photo.user_id == user_id photo_info_list.append((post, photo, user_post, user_photo)) if episode is not None and episode.viewpoint_id in viewable_viewpoint_ids: response_ep_dict = {'episode_id': ep_id} response_ep_dict.update(episode._asdict()) response_ep_dict['photos'] = [ _MakePhotoDict(photo, post, user_post, user_photo) for photo, post, user_post, user_photo in photo_info_list ] if len(photo_info_list) > 0: response_ep_dict['last_key'] = photo_info_list[-1][0].photo_id response_dict['episodes'].append(response_ep_dict) raise gen.Return(response_dict)