def test_multi_exceptions(self):
with ExpectLog(app_log, "Multiple exceptions in yield list"):
with self.assertRaises(RuntimeError) as cm:
yield gen.Multi(
[
self.async_exception(RuntimeError("error 1")),
self.async_exception(RuntimeError("error 2")),
]
)
self.assertEqual(str(cm.exception), "error 1")
# With only one exception, no error is logged.
with self.assertRaises(RuntimeError):
yield gen.Multi(
[self.async_exception(RuntimeError("error 1")), self.async_future(2)]
)
# Exception logging may be explicitly quieted.
with self.assertRaises(RuntimeError):
yield gen.Multi(
[
self.async_exception(RuntimeError("error 1")),
self.async_exception(RuntimeError("error 2")),
],
quiet_exceptions=RuntimeError,
)
def overload(n=10):
usernames = ['testuser%s' % i for i in range(n)]
while True:
tasks = [
gen.Task(db.do_request, 'PutItem', {
"Item": {
"username": {
"S": username
}
},
"TableName": "users_dev"
}) for username in usernames
]
resp = yield gen.Multi(tasks)
logging.info('puts resp %s ' % [r.code for r in resp])
tasks = [
gen.Task(db.get_item, 'users_dev', username)
for username in usernames
]
resp = yield gen.Multi(tasks)
logging.info('gets resp %s ' % [r.code for r in resp])
def test_login():
username = sys.argv[1]
password = sys.argv[2]
# check result..
#torrent = 'http://www.clearbits.net/get/503-control-alt-deus---made-of-fire.torrent'
hash = ''.join([
random.choice(list('abcdef') + map(str, range(10))) for _ in range(40)
])
torrent = 'magnet:?xt=urn:btih:%s' % hash
for _ in range(1):
client = Client(username, password)
client.sync()
yield gen.Task(asyncsleep, 1)
#client.add_url(torrent)
client.stop()
tasks = []
for hash, torrent in client.torrents.items():
if torrent.get('progress') == 1000:
tasks.append(gen.Task(torrent.fetch_files))
tasks.append(gen.Task(torrent.fetch_metadata))
responses = yield gen.Multi(tasks)
logging.info('responses %s' % [r.code for r in responses])
tasks = []
for hash, torrent in client.torrents.items():
if torrent.get('progress') == 1000:
for file in torrent.files:
link = file.webseed_link()
print link
request = tornado.httpclient.HTTPRequest(link,
validate_cert=False)
tasks.append(gen.Task(httpclient.fetch, request))
while tasks:
some_tasks = [tasks.pop() for _ in range(5)]
logging.info('executing tasks of len %s' % len(some_tasks))
responses = yield gen.Multi(some_tasks)
logging.info('responses %s' % [(r.code, len(r.body))
for r in responses])
if False:
tasks = []
for hash, torrent in client.torrents.items():
if torrent.get('progress') == 1000:
link = torrent.webseed_link()
print torrent.get('name'), torrent.get('progress'), link
request = tornado.httpclient.HTTPRequest(link,
validate_cert=False)
tasks.append(gen.Task(httpclient.fetch, request))
responses = yield gen.Multi(tasks)
logging.info('responses %s' % [r.code for r in responses])
def mul_process():
executor = concurrent.futures.ProcessPoolExecutor(
max_workers=os.cpu_count())
f_list = [executor.submit(is_prime_i, i) for i in PRIMES]
res_list = yield gen.Multi(f_list)
print(res_list)
executor.shutdown()
def f():
# callbacks run at different times
responses = yield gen.Multi([
gen.Task(self.delay_callback, 3, arg="v1"),
gen.Task(self.delay_callback, 1, arg="v2"),
])
self.assertEqual(responses, ["v1", "v2"])
self.stop()
def f():
# callbacks run at different times
responses = yield gen.Multi(dict(
foo=gen.Task(self.delay_callback, 3, arg="v1"),
bar=gen.Task(self.delay_callback, 1, arg="v2"),
))
self.assertEqual(responses, dict(foo="v1", bar="v2"))
self.stop()
def organiza_pedidos_e_envia_cozinha(self, pedidos):
"""Esta função esta responsavel por organizar os pedidos e enviar à cozinha.
Ela gera seu JSON e realiza o pedido para a cozinha.
Retorna a lista com as tasks de pedidos que estão prontos.
"""
# Criar lista de pedidos a serem preparados
pedidos_sendo_preparados = []
# Vai em cada um dos pedidos
for pedido in pedidos:
# Gera um json para se comunicar com a cozinha
pedido_json = json.dumps({"pedido": pedido})
# Realiza o pedido à cozinha
pedido_preparando = self.envia_pedido_cozinha(pedido_json)
# Adicionando o pedido a liista de pedidos à serem preparados
pedidos_sendo_preparados.append(pedido_preparando)
# Retorna uma lista de tasks dentro de uma Multi
# que sera cuidada pelo IOLoop
return gen.Multi(pedidos_sendo_preparados)
def create_domains(domains):
yield gen.Multi([gen.Task(db.create_domain, domain) for domain in domains])
def _NotifyFollowers(cls,
client,
viewpoint_id,
followers,
invalidate,
activity_func,
inc_badge=False,
always_notify=False):
"""Adds a notification for each of the given followers that the specified viewpoint has
structurally changed. If "invalidate" is a dict, then uses that directly. Otherwise, assumes
it's a function that takes a follower id and returns the invalidate dict for that follower.
If "always_notify" is true, then always send notifications, even to removed followers.
In order to minimize undesirable client artifacts caused by reads of half-committed data,
we will commit updates in this order:
1. In parallel:
a. Create the activity.
b. Update all Followed records (of all followers in the viewpoint).
2. In parallel:
a. Update update_seq in the viewpoint.
b. Update viewed_seq in the sending follower.
3. In parallel:
a. For each follower:
i. Create notification
ii. Send alert
"""
from viewfinder.backend.db.viewpoint import Viewpoint
from viewfinder.backend.op.alert_manager import AlertManager
# Get the current operation, which provides the calling user and the op timestamp.
operation = NotificationManager._GetOperation()
@gen.coroutine
def _NotifyOneFollower(viewpoint, seq_num_pair, activity, follower,
follower_settings):
"""Creates a notification for the follower and sends an alert if configured to do so."""
# If follower has been removed, do not send notifications or alerts to it.
if follower.IsRemoved() and not always_notify:
return
# Get the invalidate dict.
if invalidate is None or isinstance(invalidate, dict):
foll_invalidate = invalidate
else:
foll_invalidate = invalidate(follower.user_id)
# Don't send alert or increment badge for the user that is creating the activity and
# sending the NotificationManager.
is_sending_user = follower.user_id == operation.user_id
# Create the notification for the follower.
# Update the Followed index, which orders viewpoints by timestamp of last update.
notification = yield Notification.CreateForUser(
client,
operation,
follower.user_id,
activity.name,
invalidate=foll_invalidate,
activity_id=activity.activity_id,
viewpoint_id=viewpoint_id,
seq_num_pair=seq_num_pair,
inc_badge=inc_badge and not is_sending_user)
if not is_sending_user:
yield AlertManager.SendFollowerAlert(client, follower.user_id,
notification.badge,
viewpoint, follower,
follower_settings,
activity)
# We want a locked viewpoint while updating its sequence numbers and the corresponding Followed rows.
# Locking also prevents race conditions where new followers are added during iteration.
Viewpoint.AssertViewpointLockAcquired(viewpoint_id)
# Get affected viewpoint and the follower sending the notification, if it's available.
viewpoint = yield gen.Task(Viewpoint.Query, client, viewpoint_id, None)
sending_follower = next((follower for follower in followers
if follower.user_id == operation.user_id),
None)
# Update the viewpoint and follower sequence numbers, but do not commit until after Followed
# records are updated (since we're updating the viewpoint's "last_updated" at that time anyway).
viewpoint.update_seq += 1
if sending_follower is not None:
sending_follower.viewed_seq += 1
seq_num_pair = (viewpoint.update_seq, sending_follower.viewed_seq)
else:
seq_num_pair = (viewpoint.update_seq, None)
# Create the activity.
activity_task = activity_func(client, operation.user_id, viewpoint_id,
viewpoint.update_seq)
# Get account settings for each follower in order to determine what level of alerts they'd like.
follower_keys = [
AccountSettings.ConstructKey(follower.user_id)
for follower in followers
]
settings_task = gen.Task(AccountSettings.BatchQuery,
client,
follower_keys,
None,
must_exist=False)
# Update all Followed records.
followed_task = gen.Multi([
gen.Task(Followed.UpdateDateUpdated, client, follower.user_id,
viewpoint_id, viewpoint.last_updated, operation.timestamp)
for follower in followers
])
activity, all_follower_settings, _ = yield [
activity_task, settings_task, followed_task
]
# Now that the Followed records have been updated, update the viewpoint's "last_updated" attribute.
# This must be done afterwards so that the previous value of last_updated is known, even if the
# operation fails and restarts.
if operation.timestamp > viewpoint.last_updated:
viewpoint.last_updated = operation.timestamp
# Commit changes to update_seq and the sending follower's viewed_seq.
yield [
gen.Task(viewpoint.Update, client),
gen.Task(sending_follower.Update, client)
if sending_follower is not None else util.GenConstant(None)
]
# Visit each follower and generate notifications and alerts for it.
yield [
_NotifyOneFollower(viewpoint, seq_num_pair, activity, follower,
follower_settings) for follower,
follower_settings in zip(followers, all_follower_settings)
]
def _QueryEpisodesForArchive(client, obj_store, user_id, episode_ids):
"""Queries posts from the specified episodes.
"""
def _MakePhotoDict(post, photo, user_post, user_photo):
ph_dict = photo.MakeMetadataDict(post, user_post, user_photo)
# Do not return access URLs for posts which have been removed.
if not post.IsRemoved():
ph_dict['full_get_url'] = photo_store.GeneratePhotoUrl(
obj_store, ph_dict['photo_id'], '.f')
return ph_dict
# Get all requested episodes, along with posts for each episode.
episode_keys = [db_client.DBKey(ep_id, None) for ep_id in episode_ids]
post_tasks = []
for ep_id in episode_ids:
post_tasks.append(
gen.Task(Post.RangeQuery,
client,
ep_id,
None,
None,
None,
excl_start_key=None))
episodes, posts_list = yield [
gen.Task(Episode.BatchQuery,
client,
episode_keys,
None,
must_exist=False),
gen.Multi(post_tasks)
]
# Get viewpoint records for all viewpoints containing episodes.
viewpoint_keys = [
db_client.DBKey(viewpoint_id, None) for viewpoint_id in set(
ep.viewpoint_id for ep in episodes if ep is not None)
]
# Get follower records for all viewpoints containing episodes, along with photo and user post objects.
follower_keys = [
db_client.DBKey(user_id, db_key.hash_key) for db_key in viewpoint_keys
]
all_posts = [
post for posts in posts_list if posts is not None for post in posts
]
photo_keys = [db_client.DBKey(post.photo_id, None) for post in all_posts]
user_post_keys = [
db_client.DBKey(user_id,
Post.ConstructPostId(post.episode_id, post.photo_id))
for post in all_posts
]
if user_id:
# TODO(ben): we can probably skip this for the web view
user_photo_task = gen.Task(
UserPhoto.BatchQuery,
client,
[db_client.DBKey(user_id, post.photo_id) for post in all_posts],
None,
must_exist=False)
else:
user_photo_task = util.GenConstant(None)
viewpoints, followers, photos, user_posts, user_photos = yield [
gen.Task(Viewpoint.BatchQuery,
client,
viewpoint_keys,
None,
must_exist=False),
gen.Task(Follower.BatchQuery,
client,
follower_keys,
None,
must_exist=False),
gen.Task(Photo.BatchQuery, client, photo_keys, None),
gen.Task(UserPost.BatchQuery,
client,
user_post_keys,
None,
must_exist=False),
user_photo_task,
]
# Get set of viewpoint ids to which the current user has access.
viewable_viewpoint_ids = set(
viewpoint.viewpoint_id
for viewpoint, follower in zip(viewpoints, followers)
if _CanViewViewpointContent(viewpoint, follower))
response_dict = {'episodes': []}
for ep_id, episode, posts in zip(episode_ids, episodes, posts_list):
# Gather list of (post, photo, user_post) tuples for this episode.
photo_info_list = []
for post in posts:
photo = photos.pop(0)
user_post = user_posts.pop(0)
user_photo = user_photos.pop(
0) if user_photos is not None else None
assert photo.photo_id == post.photo_id, (episode, post, photo)
if user_photo:
assert user_photo.photo_id == photo.photo_id
assert user_photo.user_id == user_id
photo_info_list.append((post, photo, user_post, user_photo))
if episode is not None and episode.viewpoint_id in viewable_viewpoint_ids:
response_ep_dict = {'episode_id': ep_id}
response_ep_dict.update(episode._asdict())
response_ep_dict['photos'] = [
_MakePhotoDict(photo, post, user_post, user_photo)
for photo, post, user_post, user_photo in photo_info_list
]
if len(photo_info_list) > 0:
response_ep_dict['last_key'] = photo_info_list[-1][0].photo_id
response_dict['episodes'].append(response_ep_dict)
raise gen.Return(response_dict)
