def run(self, auth_pair, photos, locs=None): if not photos or not len(photos): return photoGroup = photos[0] if not locs: locs = self.getGroupLocs(auth_pair, photoGroup) # this took some time, schedule it for new run deferred.defer(self.run, auth_pair, photos, locs) return photo = photoGroup[0] loc = self.getClosestLoc(locs, photo.exif.time.text) if loc : #logging.info("updating photo: "+photo.title.text) self.updatePhoto(auth_pair, photo, loc) msg = { 'title': photo.title.text, 'thumbnail': photo.media.thumbnail[0].url, 'lat':loc['latitude'], 'lng': loc['longitude'] }; channel.send_message(auth_pair.token, json.dumps(msg)) photoGroup.pop(0) if len(photoGroup) == 0: photos.pop(0) locs = None if not len(photos) == 0: deferred.defer(self.run, auth_pair, photos, locs)
def trans(): login_state = PaymentOauthLoginState.create_key(state).get() if not login_state: return False if login_state.completed: return False login_state.completed = True login_state.put() payment_user_key = get_payment_user_key(login_state.app_user) payment_user = payment_user_key.get() if not payment_user: payment_user = PaymentUser(key=payment_user_key) payment_user.providers = [] payment_user.assets = [] pup = payment_user.get_provider(login_state.provider_id) if pup: pup.token = token else: payment_user.providers.append(PaymentUserProvider(provider_id=login_state.provider_id, token=token)) payment_user.put() deferred.defer(sync_payment_assets, login_state.app_user, login_state.provider_id, True, _transactional=True) return True
def _continue(self, start_key, batch_size): """Processes a batch of entities.""" q = self.get_query() # If we're resuming, pick up where we left off last time. if start_key: q.filter('__key__ >', start_key) # Keep updating records until we run out of time. try: # Steps over the results, returning each entity and its index. for i, entity in enumerate(q): map_updates, map_deletes = self.map(entity) self.to_put.extend(map_updates) self.to_delete.extend(map_deletes) # Record the last entity we processed. start_key = entity.key() # Do updates and deletes in batches. if (i + 1) % batch_size == 0: self._batch_write() except DeadlineExceededError: # Write any unfinished updates to the datastore. self._batch_write() # Queue a new task to pick up where we left off. defer(self._continue, start_key, batch_size) return # Write any updates to the datastore, since it may not have happened # otherwise self._batch_write() self.finish()
def _delete_non_ancestor_models(parent_service_user, service_user): sp = get_service_profile(service_user) if sp: def get_service_identity_based_keys(): keys = list() si_users = list() for si in get_service_identities(service_user): keys.append(ProfilePointer.create_key(si.service_identity_user)) si_users.append(si.service_identity_user) for qr in ServiceInteractionDef.all().ancestor(parent_key(service_user)): keys.append(db.Key.from_path(ShortURL.kind(), ServiceInteractionDef.shortUrl.get_value_for_datastore(qr).id())) return keys, si_users keys = [db.Key.from_path(Avatar.kind(), sp.avatarId)] keys.extend(TrialServiceAccount.all(keys_only=True).filter("service", service_user)) more_keys, service_identity_users = db.run_in_transaction(get_service_identity_based_keys) keys.extend(more_keys) keys.extend(MessageFlowRunRecord.all(keys_only=True).filter("service_identity >=", service_user.email() + '/').filter("service_identity <", service_user.email() + u"/\ufffd")) keys.extend(Branding.all(keys_only=True).filter("user", service_user)) keys.extend(SIKKey.all(keys_only=True).filter("user", service_user)) keys.extend(APIKey.all(keys_only=True).filter("user", service_user)) logging.info(keys) db.delete(keys) delete_service_tasks = DeleteServiceTasks(key_name=service_user.email()) delete_service_tasks.tasks = 3 delete_service_tasks.put() deferred.defer(_delete_sessions, parent_service_user, service_user) deferred.defer(_delete_service_log, parent_service_user, service_user) deferred.defer(_delete_service_models, parent_service_user, service_user) deferred.defer(_cleanup_service_identities, service_identity_users) else: if parent_service_user and parent_service_user != service_user: deferred.defer(delete_service_finished, parent_service_user, service_user.email(), True)
def run_deploy_task(): """Attempts to run the per-version deploy task.""" task_name = 'deploy-%s' % os.environ['CURRENT_VERSION_ID'].replace('.', '-') try: deferred.defer(try_post_deploy, _name=task_name, _countdown=10) except (taskqueue.TaskAlreadyExistsError, taskqueue.taskqueue.TombstonedTaskError), e: pass
def submit(): game = get_current_game() player = Player.get_by_id(session['username']) sequence = map(int, request.form.getlist('sequence[]')) g.sequence = sequence if not player.played_this_round: json_result = None player.played_this_round = True player.put() game.players_played += 1 if session['username'] == game.leader: game.sequence = g.sequence session['score'] += 1 send_channel_update('copysequence', [x.name for x in ndb.get_multi(game.players) if x.name != game.leader]) json_result = jsonify(message='Waiting on other players to copy your sequence', result='leader') else: if sequence == game.sequence: json_result = jsonify(message='Very good! Wait for the next round.', result='match') session['score'] += 1 #send_channel_update('sequencematch', [session['username']]) else: json_result = jsonify(message='You got it wrong :( oh well', result='mismatch') #send_channel_update('sequencemismatch', [session['username']]) if game.players_played == len(game.players): # next round! deferred.defer(start_round, _countdown=3) return json_result return '', 204
def send_mail_notification(subject, body, to=None, **kwargs): if not config.CONFIG_DB.feedback_email: return brand_name = config.CONFIG_DB.brand_name sender = '%s <%s>' % (brand_name, config.CONFIG_DB.feedback_email) subject = '[%s] %s' % (brand_name, subject) deferred.defer(mail.send_mail, sender, to or sender, subject, body, **kwargs)
def sendGcmToUserDeferred(irssiuser, message): logging.info("queuing deferred task for sending message to user %s" % irssiuser.email) key = irssiuser.key() try: deferred.defer(_sendGcmToUser, key, message, _queue='gcmqueue') except TransientError as e: logging.warn("Transient error: %s" % e)
def do_render_cache(cursor=None): thq = Thread.all() if cursor: thq.with_cursor(cursor) thread = thq.get() if not thread: logging.info("stop thread clean") return board = thread.parent_key().name() render = Render(board=board, thread = thread.key().id()) for idx,post in enumerate(thread.posts): post['text_html'] = markup( board=board, postid=post.get("post"), data=escape(post.get('text', '')), ) if idx == 0: render.create(post) else: render.append(post) if len(thread.posts) > 1: thread.put() else: thread.delete() render.save() deferred.defer(do_render_cache, thq.cursor())
def set(path, body, content_type, indexed=True, **kwargs): """Sets the StaticContent for the provided path. Args: path: The path to store the content against. body: The data to serve for that path. content_type: The MIME type to serve the content as. indexed: Index this page in the sitemap? **kwargs: Additional arguments to be passed to the StaticContent constructor Returns: A StaticContent object. """ content = StaticContent( key_name=path, body=body, content_type=content_type, indexed=indexed, **kwargs) content.put() memcache.replace(path, db.model_to_protobuf(content).Encode()) try: now = datetime.datetime.now().replace(second=0, microsecond=0) eta = now.replace(second=0, microsecond=0) + datetime.timedelta(seconds=65) if indexed: deferred.defer( utils._regenerate_sitemap, _name='sitemap-%s' % (now.strftime('%Y%m%d%H%M'),), _eta=eta) except (taskqueue.TaskAlreadyExistsError, taskqueue.TombstonedTaskError), e: pass
def sendGcmToTokenDeferred(token, message): logging.info("queuing deferred task for sending message to token %s" % token.gcm_token) key = token.key() try: deferred.defer(_sendGcmToToken, key, message, _queue='gcmqueue') except TransientError as e: logging.warn("Transient error: %s" % e)
def delete_shouts(self, request): # This sample runs it in the mappers backend, with 100 batches, the more things you do in map # the smaller the batch should be to avoid duplicate runs on failures deferred.defer(demo.DeleteAllShout().run, 100, _target='mappers') return message_types.VoidMessage()
def handle_upload(self): results = [] blob_keys = [] for name, fieldStorage in self.request.POST.items(): if type(fieldStorage) is unicode: continue result = {} result['name'] = re.sub(r'^.*\\','',fieldStorage.filename) result['type'] = fieldStorage.type result['size'] = self.get_file_size(fieldStorage.file) if self.validate(result): blob_key = str(self.write_blob(fieldStorage.value, result)) blob_keys.append(blob_key) result['deleteType'] = 'DELETE' result['deleteUrl'] = self.request.host_url +\ '/?key=' + urllib.quote(blob_key, '') if (IMAGE_TYPES.match(result['type'])): try: result['url'] = images.get_serving_url( blob_key, secure_url=self.request.host_url.startswith( 'https' ) ) result['thumbnailUrl'] = result['url'] +\ THUMBNAIL_MODIFICATOR except: # Could not get an image serving url pass if not 'url' in result: result['url'] = self.request.host_url +\ '/' + blob_key + '/' + urllib.quote( result['name'].encode('utf-8'), '') results.append(result) deferred.defer(cleanup,blob_keys,_countdown=EXPIRATION_TIME) return results
def _put(models,countdown=0): batch_size = 50 to_put = [] keys = [] try: last_index = 0 for i,model in enumerate(models): to_put.append(model) last_index = i if (i+1) % batch_size == 0: keys.extend(db.put(to_put)) to_put = [] keys.extend(db.put(to_put)) return keys except apiproxy_errors.DeadlineExceededError: keys.extend(db.put(to_put)) deferred.defer(_put,models[last_index+1:],_countdown=10) return keys except apiproxy_errors.CapabilityDisabledError: if not countdown: countdown = 30 else: countdown *= 2 deferred.defer(_put,models,countdown,_countdown=countdown)
def AddTestForBrowserVersions(mapping, browser_versions): """Adds tests for the given RunSiteMap the list of browser versions.""" for browser_version in browser_versions: deferred.defer(AddTestForMapping, mapping=mapping, browser_version=browser_version, _queue='tests-queue')
def post(self): self._require_admin() user_id = self.user_bundle.account.key.id() clients = self.request.get_all('client_types') title = self.request.get('title') message = self.request.get('message') url = self.request.get('url') app_version = self.request.get('app_version') error = "" if not clients: error = "clients" elif not title: error = "title" elif not message: error = "message" if error: self.redirect('/admin/mobile/broadcast?error={}'.format(error)) return try: clients = [int(c) for c in clients] deferred.defer(NotificationHelper.send_broadcast, clients, title, message, url, app_version, _queue="admin") logging.info('User {} sent broadcast'.format(user_id)) except Exception, e: logging.error("Error sending broadcast: {}".format(str(e))) logging.error("Trace: {}".format(traceback.format_exc())) self.redirect('/admin/mobile/broadcast?error=sending') return
def get(self): try: deferred.defer(src_lb_update) self.response.out.write('Task added') except TaskAlreadyExistsError: self.response.out.write('Task existed')
def txn(): try: marker = rpc.Get(self.key()) marker.__class__ = ShardedTaskMarker queued_shards = marker[ShardedTaskMarker.QUEUED_KEY] processing_shards = marker[ShardedTaskMarker.RUNNING_KEY] queued_count = len(queued_shards) for j in range(min(BATCH_SIZE, queued_count)): pickled_shard = queued_shards.pop() processing_shards.append(pickled_shard) shard = cPickle.loads(str(pickled_shard)) deferred.defer( self.run_shard, query, shard, operation, operation_method, entities_per_task=entities_per_task, # Defer this task onto the correct queue with `_queue`, passing the `queue` # parameter back to the function again so that it can do the same next time queue=queue, _queue=queue, _transactional=True, ) marker.put() except datastore_errors.EntityNotFoundError: logging.error( "Unable to start task %s as marker is missing", self.key().id_or_name() ) return
def txn(shards): marker_key = ShardedTaskMarker.get_key(identifier, query._Query__namespace) try: rpc.Get(marker_key) # If the marker already exists, don't do anything - just return return except datastore_errors.EntityNotFoundError: pass marker = ShardedTaskMarker(identifier, query, namespace=query._Query__namespace) if shards: for shard in shards: marker["shards_queued"].append(cPickle.dumps(shard)) else: # No shards, then there is nothing to do! marker["is_finished"] = True marker["time_started"] = datetime.utcnow() marker.put() if not marker["is_finished"]: deferred.defer( marker.begin_processing, operation, operation_method, entities_per_task, queue, _transactional=True, _queue=queue ) return marker_key
def start_instance(pool): """Start a new instance with a given configuration.""" # start instance # defer an update load call instance_class = string_to_class(pool.instance_type) name = rfc1035(instance_class.__name__) # max length of a name is 63 name = "{}-{}".format(name, uuid.uuid4())[:63] instance = instance_class(id=name) instance.pool = pool.key instance.zone = random.choice(LOCATIONS[pool.region]["zones"]) instance.put() compute = compute_api() if compute: def update_zone(s): return re.sub(r"\/zones\/([^/]*)", "/zones/{}".format(instance.zone), s) instance.PARAMS.update({ "name": name, "zone": update_zone(instance.PARAMS.get("zone")), "machineType": update_zone(instance.PARAMS.get("machineType")), }) operation = compute.instances().insert( project=PROJECT_ID, zone=instance.zone, body=instance.PARAMS).execute() logging.info("Create instance operation {}".format(operation)) instance.status = operation.get("status") name = "update_instance_status_{}_{}".format(instance.key.urlsafe(), int(time.time())) deferred.defer(update_instance_status, instance.key.urlsafe(), _countdown=STARTING_STATUS_DELAY, _name=name) else: logging.warn("No compute api defined.") raise AppError("No compute api defined.")
def fill_pool(pool): compute = compute_api() if compute: # find existing instances instance_class = string_to_class(pool.instance_type) name_match = ".*{}.*".format(rfc1035(instance_class.__name__)) name_filter = "name eq {}".format(name_match) size = 0 for zone in LOCATIONS[pool.region]["zones"]: resp = compute.instances().list(project=PROJECT_ID, zone=zone, filter=name_filter).execute() logging.info("List of instances {}".format(resp)) items = resp.get("items", []) for info in items: status = info.get("status") # if instance is new or running add it to the pool if status in [InstanceStatus.RUNNING, InstanceStatus.PENDING, InstanceStatus.STAGING]: logging.info("instance {}".format(info)) instance = instance_class(id=info.get("name")) instance.zone = info.get("zone").split("/")[-1] instance.status = status instance.address = info["networkInterfaces"][0]["accessConfigs"][0]["natIP"] instance.pool = pool.key instance.put() name = "update_instance_status_{}_{}".format(instance.key.urlsafe(), int(time.time())) deferred.defer(update_instance_status, instance.key.urlsafe(), _countdown=STARTING_STATUS_DELAY, _name=name) size += 1 # start any additional instances need to meet pool min_size for i in range(pool.min_size - size): LoadBalancer.start_instance(pool)
def ProposalMailer(self, action): """Notifies admins of proposed changes and changed proposals. Args: action: string, defines what message will be sent. """ current_user = users.get_current_user() current_user_nick = current_user.nickname() body = self._BuildProposalBody(os.environ.get('DEFAULT_VERSION_HOSTNAME'), self.filename) if action == 'proposal': subject = 'Proposal for %s by %s' % (self.filename, current_user_nick) elif action == 'approval': subject = 'Proposal Approved for %s by %s' % ( self.filename, current_user_nick) elif action == 'rejection': subject = 'Proposal Rejected for %s by %s' % ( self.filename, current_user_nick) else: logging.warning('Unknown action in ProposalMailer: %s', action) return recipient, _ = settings.Settings.GetItem('email_admin_list') recipient_list = [recipient, self.user] return_address, _ = settings.Settings.GetItem('email_sender') message = mail_tool.EmailMessage(to=recipient_list, sender=return_address, subject=subject, body=body) try: deferred.defer(message.send) except (deferred.Error, taskqueue.Error, apiproxy_errors.Error): logging.exception('Notification failed to send.')
def trans(): to_put = list() service_profile = get_service_profile(service_user) service_profile.expiredAt = now() service_profile.enabled = False to_put.append(service_profile) service_identity_keys = get_service_identities_query(service_user, True) search_configs = db.get( [SearchConfig.create_key(create_service_identity_user(users.User(key.parent().name()), key.name())) for key in service_identity_keys]) svc_index = search.Index(name=SERVICE_INDEX) loc_index = search.Index(name=SERVICE_LOCATION_INDEX) for search_config in search_configs: if search_config: search_config.enabled = False to_put.append(search_config) on_trans_committed(_cleanup_search_index, search_config.service_identity_user.email(), svc_index, loc_index) for objects_to_put in chunks(to_put, 200): put_and_invalidate_cache(*objects_to_put) deferred.defer(cleanup_sessions, service_user, _transactional=True) deferred.defer(cleanup_friend_connections, service_user, _transactional=True)
def trans_create(): rogerthat_profile = get_service_or_user_profile(users.User(email)) if rogerthat_profile and isinstance(rogerthat_profile, ServiceProfile): from rogerthat.bizz.service import AppFailedToCreateUserProfileWithExistingServiceException raise AppFailedToCreateUserProfileWithExistingServiceException(email) user_profile = get_user_profile(app_user, cached=False) is_new_profile = False if not user_profile: deactivated_user_profile = get_deactivated_user_profile(app_user) if deactivated_user_profile: deferred.defer(reactivate_user_profile, deactivated_user_profile, app_user, _transactional=True) ActivationLog(timestamp=now(), email=app_user.email(), mobile=None, description="Reactivate user account by registering a paper loyalty card").put() else: is_new_profile = True avatar, image = _create_new_avatar(app_user, add_trial_overlay=False) user_profile = UserProfile(parent=parent_key(app_user), key_name=app_user.email()) user_profile.name = name user_profile.language = language user_profile.avatarId = avatar.key().id() user_profile.app_id = app_id _calculateAndSetAvatarHash(user_profile, image) pp = ProfilePointer(key=db.Key.from_path(ProfilePointer.kind(), user_code)) pp.user = app_user pp.short_url_id = short_url_id if is_new_profile: put_and_invalidate_cache(user_profile, pp, ProfilePointer.create(app_user)) else: pp.put()
def trans_create(avatar, image, share_sid_key): azzert(not get_service_profile(service_user, cached=False)) azzert(not get_default_service_identity_not_cached(service_user)) profile = ServiceProfile(parent=parent_key(service_user), key_name=service_user.email()) profile.avatarId = avatar.key().id() _calculateAndSetAvatarHash(profile, image) service_identity_user = create_service_identity_user(service_user, ServiceIdentity.DEFAULT) service_identity = ServiceIdentity(key=ServiceIdentity.keyFromUser(service_identity_user)) service_identity.inheritanceFlags = 0 service_identity.name = name service_identity.description = "%s (%s)" % (name, service_user.email()) service_identity.shareSIDKey = share_sid_key service_identity.shareEnabled = False service_identity.creationTimestamp = now() service_identity.appIds = supported_app_ids update_result = update_func(profile, service_identity) if update_func else None put_and_invalidate_cache(profile, service_identity, ProfilePointer.create(service_user), ProfileHashIndex.create(service_user)) deferred.defer(create_default_qr_templates, service_user, _transactional=True) return profile, service_identity, update_result
def _QueueChangeTestPatternsAndEmail( self, entity, new_patterns, old_patterns): deferred.defer( _QueueChangeTestPatternsTasks, old_patterns, new_patterns) user_email = users.get_current_user().email() subject = 'Added or updated %s: %s by %s' % ( self._model_class.__name__, entity.key.string_id(), user_email) email_key = entity.key.string_id() email_body = _NOTIFICATION_EMAIL_BODY % { 'key': email_key, 'new_test_path_patterns': json.dumps( list(new_patterns), indent=2, sort_keys=True, separators=(',', ': ')), 'old_test_path_patterns': json.dumps( list(old_patterns), indent=2, sort_keys=True, separators=(',', ': ')), 'hostname': app_identity.get_default_version_hostname(), 'user': user_email, } mail.send_mail( sender=_SENDER_ADDRESS, to=_NOTIFICATION_ADDRESS, subject=subject, body=email_body)
def _QueueChangeTestPatternsTasks(old_patterns, new_patterns): """Updates tests that are different between old_patterns and new_patterns. The two arguments both represent sets of test paths (i.e. sets of data series). Any tests that are different between these two sets need to be updated. Some properties of TestMetadata entities are updated when they are put in the |_pre_put_hook| method of TestMetadata, so any TestMetadata entity that might need to be updated should be re-put. Args: old_patterns: An iterable of test path pattern strings. new_patterns: Another iterable of test path pattern strings. Returns: A pair (added_test_paths, removed_test_paths), which are, respectively, the test paths that are in the new set but not the old, and those that are in the old set but not the new. """ added_patterns, removed_patterns = _ComputeDeltas(old_patterns, new_patterns) patterns = list(added_patterns) + list(removed_patterns) for i in xrange(0, len(patterns), _NUM_PATTERNS_PER_TASK): pattern_sublist = patterns[i:i+_NUM_PATTERNS_PER_TASK] deferred.defer(_GetTestPathsAndAddTask, pattern_sublist)
def do_clean(cursor=None): bq = BlobInfo.all() if cursor: bq.with_cursor(cursor) blob = bq.get() if not blob: return key = str(blob.key()) thq = Thread.all(keys_only=True) thq.filter("images", key) th = thq.get() if th: logging.info("thread: %r" % th) else: logging.info("no thread for image %r" % key) blob.delete() deferred.defer(do_clean, bq.cursor(), _countdown=30)
def _delete_service_log(parent_service_user, service_user): keys = ServiceLog.all(keys_only=True).filter("user", service_user).fetch(1000) if keys: db.delete(keys) deferred.defer(_delete_service_log, parent_service_user, service_user, _countdown=5) else: _decrease_and_verify(parent_service_user, service_user)
def view_that_defers(request): from google.appengine.ext.deferred import defer from django.http import HttpResponse defer(do_something) return HttpResponse("OK")
def get(self): deferred.defer(controller.delete_all_step_records, _queue='summary-delete', _target='stats-backend') self.response.out.write('deleted all')
youtube_id=youtube_id, language=lang, json=json) new.put() report['writes'] += 1 else: logging.info('%s content already up-to-date' % key_name) except Exception, e: logging.error('%s subtitles fetch failed: %s' % (key_name, e)) report['errors'] += 1 # Generate a report if there is nothing left to process if len(videos) < BATCH_SIZE: deferred.defer(_task_report_handler, uid, report, _name='%s_report' % uid, _queue=TASK_QUEUE) else: next_id = task_id + 1 cursor = query.cursor() deferred.defer(_task_handler, uid, next_id, cursor, report, _name='%s_%s' % (uid, next_id), _queue=TASK_QUEUE, _countdown=DEFER_SECONDS)
def start(self): """ Don't do anything in a time constrained view. Kick of the first deferred task. """ deferred.defer(self.map, _queue=self.queue)
def process_notification(notification, now, stats): notification_key = notification.key() policy = None stats.started += 1 # Treat as module-protected. pylint: disable-msg=protected-access if notification._done_date: _LOG.info( 'Skipping offline processing of notification with key %s; already ' 'done at %s', notification_key, notification._done_date) stats.skipped_already_done += 1 return if notifications.Manager._is_still_enqueued(notification, now): _LOG.info( 'Skipping offline processing of notification with key %s; still on ' 'queue (last enqueued: %s)', notification_key, notification._last_enqueue_date) stats.skipped_still_enqueued += 1 return payload_key = db.Key.from_path( notifications.Payload.kind(), notifications.Payload.key_name(notification.to, notification.intent, notification.enqueue_date)) payload = db.get(payload_key) if not payload: _LOG.error( 'Could not process notification with key %s; associated payload with ' 'key %s not found', notification_key, payload_key) stats.missing_payload += 1 return if notifications.Manager._is_too_old_to_reenqueue( notification.enqueue_date, now): stats.too_old += 1 exception = notifications.NotificationTooOldError(( 'Notification %s with enqueue_date %s too old to re-enqueue at %s; ' 'limit is %s days') % ( notification_key, notification.enqueue_date, now, notifications._MAX_RETRY_DAYS, )) notifications.Manager._mark_failed(notification, now, exception, permanent=True) if notification._fail_date or notification._send_date: policy = notifications._RETENTION_POLICIES.get( notification._retention_policy) notifications.Manager._mark_done(notification, now) if policy: policy.run(notification, payload) stats.policy_run += 1 else: _LOG.warning( 'Cannot apply retention policy %s to notification %s and payload ' '%s; policy not found. Existing policies are: %s', notification._retention_policy, notification_key, payload_key, ', '.join(sorted(notifications._RETENTION_POLICIES.keys()))) stats.missing_policy += 1 db.put([notification, payload]) else: notifications.Manager._mark_enqueued(notification, now) db.put(notification) deferred.defer( notifications.Manager._transactional_send_mail_task, notification_key, payload_key, _retry_options=notifications.Manager._get_retry_options()) stats.reenqueued += 1
def post(self): data = json.loads(self.request.body) name, phone, email = map(data['info'].get, ('name', 'phone', 'email')) phone = '7' + ''.join(c for c in phone if '0' <= c <= '9') namespace = self._find_namespace(name) password = "******" % random.randint(0, 10000) CompanyUser.create_user(namespace, namespace=namespace, password_raw=password, login=namespace, rights=CompanyUser.RIGHTS_MASK_ADMIN) namespace_manager.set_namespace(namespace) cfg = Config(id=1) cfg.APP_NAME = name cfg.DELIVERY_PHONES = [phone] cfg.DELIVERY_EMAILS = ['*****@*****.**', email] cfg.SUPPORT_EMAILS = [email] cfg.ACTION_COLOR = "FF25B8CD" cfg.put() delivery_slot_keys = [ DeliverySlot(name=u'Сейчас', slot_type=0, value=0).put(), DeliverySlot(name=u'Через 5 минут', slot_type=0, value=5).put(), DeliverySlot(name=u'Через 10 минут', slot_type=0, value=10).put(), DeliverySlot(name=u'Через 15 минут', slot_type=0, value=15).put(), DeliverySlot(name=u'Через 20 минут', slot_type=0, value=20).put(), DeliverySlot(name=u'Через 25 минут', slot_type=0, value=25).put(), DeliverySlot(name=u'Через 30 минут', slot_type=0, value=30).put() ] menu = data['menu'] init = MenuCategory(category=None, title="Init") init.put() for i, category_dict in enumerate(menu): MenuCategory.generate_category_sequence_number( ) # only to increase counter category = MenuCategory(title=category_dict["title"], sequence_number=i, category=init.key) category.put() for j, item in enumerate(category_dict["items"]): MenuItem(category=category.key, title=item["title"], description=item["description"], picture=item["imageUrl"], price=int(round(float(item["price"]) * 100)), sequence_number=j).put() for _ in category.get_items(): category.generate_sequence_number() # only to increase counter venue_dict = data['venue'] venue = Venue(title=venue_dict['title'], description=venue_dict['address'], coordinates=ndb.GeoPt(venue_dict['lat'], venue_dict['lng']), active=True) venue.update_address() venue.schedule = Schedule(days=[ DaySchedule( weekday=i, start=datetime.time(0, 0), end=datetime.time(0, 0)) for i in xrange(1, 8) ]) for delivery_type in (SELF, IN_CAFE): delivery = DeliveryType.create(delivery_type) delivery.put() delivery.status = STATUS_AVAILABLE delivery.max_time = DAY_SECONDS + HOUR_SECONDS # need hour to order on tomorrow delivery.delivery_slots = delivery_slot_keys venue.delivery_types.append(delivery) venue.put() PaymentType(id=str(CASH_PAYMENT_TYPE), title="cash", status=STATUS_AVAILABLE).put() deferred.defer(_notify_sms, phone, namespace, password) deferred.defer(_notify_email, email, phone, name, namespace, password) self.render_json({'login': namespace, 'password': password})
def get(self): deferred.defer(controller.step_cleanup, _queue='step-operations', _target='stats-backend') self.response.out.write('steps cleaned')
def get(self): deferred.defer(controller.get_steps, disable_cache_lookup=True, _queue='step-operations', _target='stats-backend') self.response.out.write('steps cached')
def update_user_data(sik, service_identity, email, app_id, address, notifications, collections): deferred.defer(_update_user_data, sik, service_identity, email, app_id, address, notifications, collections, _transactional=ndb.in_transaction())
def handle_upload(self): print "test" results = [] blob_keys = [] for name, fieldStorage in self.request.POST.items(): if type(fieldStorage) is unicode: continue result = {} result['name'] = re.sub( r'^.*\\', '', fieldStorage.filename ) result['type'] = fieldStorage.type result['size'] = self.get_file_size(fieldStorage.file) if self.validate(result): blob_key = str( self.write_blob(fieldStorage.value, result) ) blob_keys.append(blob_key) result['deleteType'] = 'DELETE' result['deleteUrl'] = self.request.host_url +\ '/?key=' + urllib.quote(blob_key, '') #image = Image(parent=streamKey) #upload_files = self.get_uploads('file') #if (len(upload_files) > 0 ): # blob_info = upload_files[0] # website = images.get_serving_url(blob_info) # image.blob_url = website # image.blob_key = blob_info.key() # image.comment = self.request.get("comments") # image.put() # stream_obj.date_updated = datetime.datetime.now() # stream_obj.date_only_updated = datetime.datetime.now().date() # stream_obj.num_pics = stream_obj.num_pics + 1 # stream_obj.put() #query_params = {'stream_key': stream_obj.key.urlsafe(), 'upload': True} #self.redirect('/streamview?' + urllib.urlencode(query_params)) if (IMAGE_TYPES.match(result['type'])): try: print "hello" result['url'] = images.get_serving_url( blob_key, secure_url=self.request.host_url.startswith( 'https' ) ) print result['url'] result['thumbnailUrl'] = result['url'] +\ THUMBNAIL_MODIFICATOR print str(fieldStorage.file) print str(fieldStorage) print str(fieldStorage.type) print blob_key urlString = self.request.get("stream_key") streamKey = ndb.Key(urlsafe=urlString) image = Image(parent=streamKey) image.blob_url = result['url'] key = str(blob_key) print key image.blob_key = BlobKey(key) image.comment = "" long = uniform(-180,180) lat = uniform(-70, 70) image.longitude = long image.latitude = lat image.put() stream_obj = streamKey.get() stream_obj.date_updated = datetime.datetime.now() stream_obj.date_only_updated = datetime.datetime.now().date() stream_obj.num_pics = stream_obj.num_pics + 1 stream_obj.put() except: # Could not get an image serving url pass if not 'url' in result: result['url'] = self.request.host_url +\ '/' + blob_key + '/' + urllib.quote( result['name'].encode('utf-8'), '') results.append(result) deferred.defer( cleanup, blob_keys, _countdown=EXPIRATION_TIME ) return results
def send_collection_message(sik, service_identity, email, app_id, message): json_rpc_id = guid() deferred.defer(_send_collection_message, sik, service_identity, email, app_id, message, json_rpc_id, _transactional=ndb.in_transaction())
def check_bq_job(job_id, item_ids, suggestions_key, page_token): bq = BigQueryClient() logging.info("Polling suggestion job {}.".format(job_id)) # TODO: catch 404 errors for jobs created 24+ hours ago, retry with new jobid try: bq_json = bq.get_async_job_results(job_id, page_token, MAX_RESULTS_PER_SUGGESTIONS_QUERY) except Exception as e: logging.error("Error from BigQuery with item_id={}.".format(item_ids)) raise deferred.PermanentTaskFailure(e) if not bq_json['jobComplete']: logging.info("- job not completed yet.") deferred.defer(check_bq_job, job_id, item_ids, suggestions_key, "", _countdown=5) return if not 'rows' in bq_json: logging.error( u"Invalid json for BigQueryTable. Job for {} is probably " u"invalid (bad item_id?).\n" u"JSON:\n" u"{}".format(item_ids, bq_json)) raise deferred.PermanentTaskFailure("No rows in BigQuery response for " "{}.".format(item_ids)) table = BigQueryTable(bq_json) item_ids_array = item_ids.split('|') # Get the consolidated book for each item_id suggestions = suggestions_key.get() assert isinstance(suggestions, SuggestionsRecord) for row in table.data: item_id = row[0] prediction = float(row[1]) if item_id in item_ids_array: continue # Original book. consolidated_book_key = BookRecord.query( BookRecord.item_id_array == item_id).get(keys_only=True) if not consolidated_book_key: logging.info( "No consolidated book with item_id '{}' found.".format( item_id)) continue if not consolidated_book_key in suggestions.books: suggestions.books.append(consolidated_book_key) suggestions.books_prediction.append(prediction) if len(suggestions.books) >= 1000: break next_page_token = bq_json.get('pageToken', "") if next_page_token != "" and len(suggestions.books) < 1000: suggestions.put() deferred.defer(check_bq_job, job_id, item_ids, suggestions_key, next_page_token) logging.info("Suggestions for item_ids '{}' partly fetched. " "Running again.".format(item_ids)) else: suggestions.completed = True suggestions.json_generation_started = True suggestions.put() logging.info( "Suggestions for item_ids '{}' completed and saved.".format( item_ids)) deferred.defer(create_suggestions_json, suggestions.key)
def get(self): """Rollup logs into weeks""" deferred.defer(rollup_events, _queue='rollup')
def change_view(self, handler, object_id, extra_context=None): """View zum Bearbeiten eines vorhandenen Objekts""" obj = self.get_object(object_id) if obj is None: raise gaetk.handler.HTTP404_NotFound model_class = type(obj) form_class = self.get_form() if handler.request.get('delete') == 'yesiwant': # Der User hat gebeten, dieses Objekt zu löschen. key = compat.xdb_key(obj) data = compat.xdb_to_protobuf(obj) dblayer = 'ndb' if compat.xdb_is_ndb(obj) else 'db' archived = DeletedObject(key_name=str(key), model_class=model_class.__name__, old_key=str(key), dblayer=dblayer, data=data) archived.put() # Indexierung für Admin-Volltextsuche from gaetk.admin.search import remove_from_index if compat.xdb_is_ndb(obj): obj.key.delete() deferred.defer(remove_from_index, obj.key) else: obj.delete() deferred.defer(remove_from_index, obj.key()) handler.add_message( 'warning', u'<strong>{} {}</strong> wurde gelöscht. <a href="{}">Objekt wiederherstellen!</a>' .format(compat.xdb_kind(self.model), compat.xdb_id_or_name(key), archived.undelete_url())) raise gaetk.handler.HTTP302_Found( location='/admin/%s/%s/' % (util.get_app_name(model_class), compat.xdb_kind(model_class))) # Wenn das Formular abgeschickt wurde und gültig ist, # speichere das veränderte Objekt und leite auf die Übersichtsseite um. if handler.request.method == 'POST': form = form_class(handler.request.POST) if form.validate(): key_name = compat.xdb_id_or_name(compat.xdb_key(obj)) self.handle_blobstore_fields(handler, obj, key_name) if hasattr(obj, 'update'): obj.update(form.data) else: form.populate_obj(obj) key = obj.put() handler.add_message( 'success', u'<strong><a href="/admin/{}/{}/{}/">{} {}</a></strong> wurde gespeichert.' .format(util.get_app_name(self.model), compat.xdb_kind(self.model), compat.xdb_str_key(key), compat.xdb_kind(self.model), compat.xdb_id_or_name(key))) # Indexierung für Admin-Volltextsuche from gaetk.admin.search import add_to_index deferred.defer(add_to_index, key) raise gaetk.handler.HTTP302_Found( location='/admin/%s/%s/' % (util.get_app_name(model_class), compat.xdb_kind(model_class))) else: form = form_class(obj=obj) template_values = { 'object': obj, 'form': form, 'field_args': self.field_args, 'admin_class': self } if extra_context is not None: template_values.update(extra_context) handler.render(template_values, self.get_template('change'))
def put(self): isNew = not self.is_saved() db.Model.put(self) self.putSearchResult() isNew and deferred.defer(SiteCounters.incrementCounter, 'campaigns')
def get(self): event = self.request.get('event', 'test') deferred.defer(test_stream.stream.populate, 'timedtext.xml', event)
def add_view(self, handler, extra_context=None): """View zum Hinzufügen eines neuen Objekts""" form_class = self.get_form() # Standardmaessig lassen wir die App Engine fuer das Model automatisch einen # Key generieren. Es besteht jedoch in der Admin-Klasse die Moeglichkeit, via # 'db_key_field=[propertyname]' ein Feld festzulegen, dessen Inhalt im Formular # als Key beim Erzeugen der Instanz genutzt wird. admin_class = site.get_admin_class(self.model) key_field = None if admin_class and hasattr(admin_class, 'db_key_field'): key_field = admin_class.db_key_field # Wenn das Formular abgeschickt wurde und gültig ist, # speichere das veränderte Objekt und leite auf die Übersichtsseite um. if handler.request.method == 'POST': form = form_class(handler.request.POST) if form.validate(): form_data = self._convert_property_data(form.data) key_name = form_data.get(key_field) if key_field else None # TODO: util.create_instance nutzen oder entfernen if hasattr(self.model, 'create'): factory = self.model.create else: factory = self.model if issubclass(self.model, ndb.Model): obj = factory(id=key_name, **form_data) else: obj = factory(key_name=key_name, **form_data) # Beim Anlegen muss dann halt einmal gespeichert werden, # ansonsten ist der ID unbekannt. if self.blob_upload_fields and key_name is None: key_name = compat.xdb_id_or_name(obj.put()) self.handle_blobstore_fields(handler, obj, key_name) key = obj.put() handler.add_message( 'success', u'<strong><a href="/admin/{}/{}/{}/">{} {}</a></strong> wurde angelegt.' .format(util.get_app_name(self.model), compat.xdb_kind(self.model), compat.xdb_str_key(key), compat.xdb_kind(self.model), compat.xdb_id_or_name(key))) # Indexierung für Admin-Volltextsuche from gaetk.admin.search import add_to_index deferred.defer(add_to_index, key) # Call post-create-hooks if isinstance(self.post_create_hooks, collections.Iterable): for hook in self.post_create_hooks: deferred.defer(util.call_hook, hook, compat.xdb_str_key(key)) raise gaetk.handler.HTTP302_Found(location='..') else: form = form_class() template_values = { 'form': form, 'field_args': self.field_args, 'admin_class': self } if extra_context is not None: template_values.update(extra_context) handler.render(template_values, self.get_template('add'))
def _FindAndRestartJobs(): jobs = _FindFrozenJobs() opts = TaskRetryOptions(task_retry_limit=1) for j in jobs: deferred.defer(_ProcessFrozenJob, j.job_id, _retry_options=opts)
def Send(self, to, subject, text_body, html_body, reply_to=None): if self.defer: deferred.defer(_send_mail, to, subject, text_body, html_body, reply_to) else: _send_mail(to, subject, text_body, html_body, reply_to)
def get(self): for bm in Bookmarks.query(): deferred.defer(add_domain, bm.key, _target="worker", _queue="admin")
def defer(callable, *args, **kwargs): if ndb.in_transaction() and kwargs.get('_transactional') is not False: kwargs['_transactional'] = True deferred.defer(callable, *args, **kwargs)
def get(self): for feed in Feeds.query(): if feed.notify == 'digest': deferred.defer(utils.feed_digest, feed.key, _target="worker", _queue="admin")
def get(self): feed = Feeds.get_by_id(int(self.request.get('feed'))) deferred.defer(utils.pop_feed, feed.key, _queue="admin")
def DeferCommitBlockableChangeSet( blockable_key, tail_defer=True, tail_defer_count=0, countdown=0): deferred.defer( _CommitBlockableChangeSet, blockable_key, tail_defer=tail_defer, tail_defer_count=tail_defer_count, _queue=constants.TASK_QUEUE.BIT9_COMMIT_CHANGE, _countdown=countdown)
def get(self): for ui in UserInfo.query(): if ui.daily: deferred.defer(utils.daily_digest, ui.user, _target="worker", _queue="admin")
def post(self): deferred.defer(self._get_cmd().run) self.response.write('Command started.')
def get(self): for feed in Feeds.query(): deferred.defer(utils.pop_feed, feed.key, _target="worker", _queue="admin")
def Midpoint(cls, commit_a, commit_b): """Return a Commit halfway between the two given Commits. Uses Gitiles to look up the commit range. Args: commit_a: The first Commit in the range. commit_b: The last Commit in the range. Returns: A new Commit representing the midpoint. The commit before the midpoint if the range has an even number of commits. commit_a if the Commits are the same or adjacent. Raises: NonLinearError: The Commits are in different repositories or commit_a does not come before commit_b. """ if commit_a == commit_b: return commit_a if commit_a.repository != commit_b.repository: raise NonLinearError( 'Repositories differ between Commits: %s vs %s' % (commit_a.repository, commit_b.repository)) # We need to get the full list of commits in between two git hashes, and # only look into the chain formed by following the first parents of each # commit. This gives us a linear view of the log even in the presence of # merge commits. commits = [] # The commit_range by default is in reverse-chronological (latest commit # first) order. This means we should keep following the first parent to get # the linear history for a branch that we're exploring. expected_parent = commit_b.git_hash commit_range = gitiles_service.CommitRange(commit_a.repository_url, commit_a.git_hash, commit_b.git_hash) for commit in commit_range: # Skip commits until we find the parent we're looking for. if commit['commit'] == expected_parent: commits.append(commit) if 'parents' in commit and len(commit['parents']): expected_parent = commit['parents'][0] # We don't handle NotFoundErrors because we assume that all Commits either # came from this method or were already validated elsewhere. if len(commits) == 0: raise NonLinearError( 'Commit "%s" does not come before commit "%s".' % (commit_a, commit_b)) if len(commits) == 1: return commit_a commits.pop(0) # Remove commit_b from the range. deferred.defer(_CacheCommitDetails, commit_a.repository, commits[len(commits) / 2]['commit']) return cls(commit_a.repository, commits[len(commits) / 2]['commit'])
def tell_queue(chat_id, msg, kb=None, markdown=True, inlineKeyboardMarkup=False, one_time_keyboard=True): deferred.defer(tell,chat_id, msg, kb=kb, markdown=markdown, inlineKeyboardMarkup=inlineKeyboardMarkup, one_time_keyboard=one_time_keyboard, _queue="messages-queue")
def pull_data(user, third_party_user): deferred.defer(pull_shots, user, third_party_user)
def close_channel(channel_id): # Sets /channels/<channel_id>/_meta/status to "closed", and schedules a # deferred task to remove_channel. result = fb_put("/channels/" + channel_id + "/_meta/status", "closed") deferred.defer(remove_channel, channel_id, _countdown=30)