def make_session(userinfo, expiration_sec): """Creates new AuthOpenIDSession (and AuthOpenIDUser if needed) entities. Args: userinfo: user profile dict as returned by handle_authorization_code. expiration_sec: how long (in seconds) the session if allowed to live. Returns: AuthOpenIDSession already persisted in the datastore. """ now = utils.utcnow() # Refresh datastore entry for logged in user. user = AuthOpenIDUser( id=userinfo["sub"].encode("ascii"), last_session_ts=now, email=userinfo["email"], name=userinfo["name"], picture=userinfo["picture"], ) # Create a new session that expires at the same time when cookie signature # expires. ID is autogenerated by the datastore. session = AuthOpenIDSession( parent=user.key, created_ts=now, expiration_ts=now + datetime.timedelta(seconds=expiration_sec), email=user.email, name=user.name, picture=user.picture, ) ndb.transaction(lambda: ndb.put_multi([user, session])) assert session.key.integer_id() return session
def test_get_remove_identifiers(self): """ Tests the get and remove by identifier functions """ tree = MultiBTree2.create("tree", 3) tree.update((1, str(x), str(x)) for x in range(30)) for x in range(30): self.assertEqual((1, str(x), str(x)), tree.get_by_identifier(str(x))) for x in range(30): tree.remove_by_identifier(str(x)) self.assertEqual(None, tree.get_by_identifier(str(x))) self.assertEqual(tree.tree_size(), 0) item = (10, "10", "id") item2 = (11, "11", "id2") def txn(): tree.update([item, item2]) self.assertEqual(10 in tree, True) self.assertEqual(11 in tree, True) self.assertEqual(tree.get_by_identifier("id"), item) self.assertEqual(tree.get_by_identifier("id2"), item2) tree.remove_by_identifier("id") self.assertEqual(tree.get_by_identifier("id"), None) ndb.transaction(txn) self.assertEqual(tree.tree_size(), 1) self.assertEqual(tree.get_by_identifier("id"), None) self.assertEqual(tree.get_by_identifier("id2"), item2) self.validate_indices(tree)
def post(self): from libs.feedparser import parse user = users.get_current_user() url = self.request.get('url') p = parse(str(url)) try: d = p['items'][0] except IndexError: pass if user: q = Feeds.query(Feeds.user == user, Feeds.url == url) if q.get() is None: feed = Feeds() def txn(): feed.blog = p.feed.title feed.root = p.feed.link feed.user = user feed.feed = url feed.url = d.link feed.put() ndb.transaction(txn) deferred.defer(utils.new_bm, d, feed.key, _queue="admin") self.redirect(self.request.referer) else: self.redirect('/')
def _order_node_iyo_see(app_user, node_order_id, pdf_url, pdf_size, create_quotation=True): iyo_username = get_iyo_username(app_user) doc_id = u'Zero-Node order %s' % NodeOrder.create_human_readable_id( node_order_id) category = u'Terms and conditions' content_type = u'application/pdf' description = u'Terms and conditions for ordering a Zero-Node' create_see_document(doc_id, category, description, iyo_username, pdf_url, content_type) attachment_name = u' - '.join([doc_id, category]) def trans(): order = get_node_order(node_order_id) order.tos_iyo_see_id = doc_id order.put() if create_quotation: deferred.defer(_create_quotation, app_user, node_order_id, pdf_url, attachment_name, pdf_size, _transactional=True) ndb.transaction(trans)
def save(self, must_create=False): """Create and save a Session object using db.run_in_transaction, with key_name = session_key, raising CreateError if unsuccessful. """ if must_create: s = self.get_ndb_session_key().get() if s: raise CreateError() session_data = self._get_session(no_load=must_create) #ed = self.get_expiry_date() #print datetime.datetime.utcoffset(ed) def txn(): s = Session(id=self._get_or_create_session_key(), session_key=self.session_key, session_data=self.encode(session_data), expire_date=self.get_expiry_date()) s.put() # This is tricky and probably needs some sanity checking, because # TransactionFailedError can be raised, but the transaction can still # go on to be committed to the datastore. As far as I can see there's # no way to manually roll it back at that point. No idea how to test # this either. try: ndb.transaction(txn) except (ndb.Rollback): raise CreateError()
def test_new_result_summary(self): request = _gen_request() actual = task_result.new_result_summary(request) actual.modified_ts = self.now # Trigger _pre_put_hook(). actual.put() expected = self._gen_summary(modified_ts=self.now) self.assertEqual(expected, actual.to_dict()) self.assertEqual(50, actual.request.priority) self.assertEqual(True, actual.can_be_canceled) actual.state = task_result.State.RUNNING self.assertEqual(True, actual.can_be_canceled) actual.state = task_result.State.TIMED_OUT actual.duration = 0.1 self.assertEqual(False, actual.can_be_canceled) actual.children_task_ids = [ '1d69ba3ea8008810', '3d69ba3ea8008810', '2d69ba3ea8008810', ] actual.modified_ts = utils.utcnow() ndb.transaction(actual.put) expected = [ u'1d69ba3ea8008810', u'2d69ba3ea8008810', u'3d69ba3ea8008810' ] self.assertEqual(expected, actual.key.get().children_task_ids)
def updateAverageRating(review_key): """Helper function for updating the average rating of a product when new review(s) are added.""" def _tx(): review = review_key.get() v = review.video_key.get() if not review.rating_added: review.rating_added = True v.num_reviews += 1 v.avg_rating = ( v.avg_rating + (review.rating - v.avg_rating) / float(v.num_reviews)) # signal that we need to reindex the doc with the new ratings info. v.needs_review_reindex = True ndb.put_multi([v, review]) # We need to update the ratings associated document at some point as well. # If the app is configured to have BATCH_RATINGS_UPDATE set to True, don't # do this re-indexing now. (Instead, all the out-of-date documents can be # be later handled in batch -- see cron.yaml). If BATCH_RATINGS_UPDATE is # False, go ahead and reindex now in a transational task. if not config.BATCH_RATINGS_UPDATE: defer(models.Video.updateVideoDocWithNewRating, v.key.id(), _transactional=True) return (v, review) try: # use an XG transaction in order to update both entities at once ndb.transaction(_tx, xg=True) except AttributeError: # swallow this error and log it; it's not recoverable. logging.exception( 'The function updateAverageRating failed. Either review ' + 'or product entity does not exist.')
def updateProdDocsWithNewRating(cls, pkeys): """Given a list of product entity keys, check each entity to see if it is marked as needing a document re-index. This flag is set when a new review is created for that product, and config.BATCH_RATINGS_UPDATE = True. Generate the modified docs as needed and batch re-index them.""" doclist = [] def _tx(pid): prod = cls.get_by_id(pid) if prod and prod.needs_review_reindex: # update the associated document with the new ratings info # and reindex modified_doc = docs.Product.updateRatingInDoc( prod.doc_id, prod.avg_rating) if modified_doc: doclist.append(modified_doc) prod.needs_review_reindex = False prod.put() for pkey in pkeys: ndb.transaction(lambda: _tx(pkey.id())) # reindex all modified docs in batch docs.Product.add(doclist)
def investment_agreement_signed_by_admin(status, form_result, answer_id, member, message_key, tag, received_timestamp, acked_timestamp, parent_message_key, result_key, service_identity, user_details): tag_dict = json.loads(tag) def trans(): agreement = InvestmentAgreement.create_key(tag_dict['agreement_id']).get() # type: InvestmentAgreement if answer_id != FormTO.POSITIVE: logging.info('Investment agreement sign aborted') return if agreement.status == InvestmentAgreement.STATUS_PAID: logging.warn('Ignoring request to set InvestmentAgreement %s as paid because it is already paid', agreement.id) return agreement.status = InvestmentAgreement.STATUS_PAID agreement.paid_time = now() agreement.put() user_email, app_id, = get_app_user_tuple(agreement.app_user) deferred.defer(transfer_genesis_coins_to_user, agreement.app_user, TokenType.I, long(agreement.token_count_float * 100), _transactional=True) deferred.defer(update_investor_progress, user_email.email(), app_id, INVESTMENT_TODO_MAPPING[agreement.status], _transactional=True) deferred.defer(_send_tokens_assigned_message, agreement.app_user, _transactional=True) ndb.transaction(trans)
def _update_transaction_with_payconiq_info(transaction_id, payconic_transaction_id): def trans(): transaction = PayconiqTransaction.create_key(transaction_id).get() transaction.payconic_transaction_id = payconic_transaction_id transaction.put() ndb.transaction(trans)
def updateAverageRating(review_key): """Helper function for updating the average rating of a product when new review(s) are added.""" def _tx(): review = review_key.get() product = review.product_key.get() if not review.rating_added: review.rating_added = True product.num_reviews += 1 product.avg_rating = (product.avg_rating + (review.rating - product.avg_rating)/float(product.num_reviews)) # signal that we need to reindex the doc with the new ratings info. product.needs_review_reindex = True ndb.put_multi([product, review]) # We need to update the ratings associated document at some point as well. # If the app is configured to have BATCH_RATINGS_UPDATE set to True, don't # do this re-indexing now. (Instead, all the out-of-date documents can be # be later handled in batch -- see cron.yaml). If BATCH_RATINGS_UPDATE is # False, go ahead and reindex now in a transational task. if not config.BATCH_RATINGS_UPDATE: defer( models.Product.updateProdDocWithNewRating, product.key.id(), _transactional=True) return (product, review) try: # use an XG transaction in order to update both entities at once ndb.transaction(_tx, xg=True) except AttributeError: # swallow this error and log it; it's not recoverable. logging.exception('The function updateAverageRating failed. Either review ' + 'or product entity does not exist.')
def test_set_from_run_result_two_server_versions(self): request = _gen_request() result_summary = task_result.new_result_summary(request) to_run = task_to_run.new_task_to_run(request, 1, 0) run_result = task_result.new_run_result(request, to_run, 'localhost', 'abc', {}) run_result.started_ts = utils.utcnow() self.assertTrue(result_summary.need_update_from_run_result(run_result)) result_summary.modified_ts = utils.utcnow() run_result.modified_ts = utils.utcnow() ndb.transaction(lambda: ndb.put_multi((result_summary, run_result))) self.assertTrue(result_summary.need_update_from_run_result(run_result)) ndb.transaction( lambda: result_summary.set_from_run_result(run_result, request)) ndb.transaction(lambda: ndb.put_multi([result_summary])) run_result.signal_server_version('new-version') run_result.modified_ts = utils.utcnow() ndb.transaction( lambda: result_summary.set_from_run_result(run_result, request)) ndb.transaction(lambda: ndb.put_multi((result_summary, run_result))) self.assertEqual(['v1a', 'new-version'], run_result.key.get().server_versions) self.assertEqual(['v1a', 'new-version'], result_summary.key.get().server_versions)
def save(self, must_create=False): """Create and save a Session object using db.run_in_transaction, with key_name = session_key, raising CreateError if unsuccessful. """ if must_create: s = self.get_ndb_session_key().get() if s: raise CreateError() session_data = self._get_session(no_load=must_create) #ed = self.get_expiry_date() #print datetime.datetime.utcoffset(ed) def txn(): s = Session( id=self._get_or_create_session_key(), session_key=self.session_key, session_data=self.encode(session_data), expire_date=self.get_expiry_date() ) s.put() # This is tricky and probably needs some sanity checking, because # TransactionFailedError can be raised, but the transaction can still # go on to be committed to the datastore. As far as I can see there's # no way to manually roll it back at that point. No idea how to test # this either. try: ndb.transaction(txn) except (ndb.Rollback): raise CreateError()
def backfill_tags(entity): # Already handled? if entity.tags: return # TaskRequest is immutable, can be fetched outside the transaction. task_request = entity.request_key.get(use_cache=False, use_memcache=False) if not task_request or not task_request.tags: return # Fast path for old entries: do not use transaction, assumes old entities are # not being concurrently modified outside of this job. if entity.created_ts and entity.created_ts < OLD_TASKS_CUTOFF: entity.tags = task_request.tags yield operation.db.Put(entity) return # For recent entries be careful and use transaction. def fix_task_result_summary(): task_result_summary = entity.key.get() if task_result_summary and not task_result_summary.tags: task_result_summary.tags = task_request.tags task_result_summary.put() ndb.transaction(fix_task_result_summary, use_cache=False, use_memcache=False)
def set_vote(quote_id, user_id, newvote, provider): if user_id is None: return if isinstance(user_id, unicode): auth_id = '%s:%s' % (provider, user_id) user = User.get_by_auth_id(auth_id) else: user = user_id email = user.email def txn(): quote = Post.get_by_id(quote_id) vote = Vote.get_by_id(id=email, parent=quote.key) if vote is None: vote = Vote(id=email, parent=quote.key) if vote.vote == newvote: return quote.votesum = quote.votesum - vote.vote + newvote vote.vote = newvote quote.rank = "%020d|%s" % ( long(quote.created * DAY_SCALE + quote.votesum), quote.creation_order ) quote.put() vote.put() memcache.set("vote|" + email + "|" + str(quote_id), vote.vote) ndb.transaction(txn)
def _order_node_iyo_see(app_user, node_order_id, pdf_url): iyo_username = get_iyo_username(app_user) organization_id = get_iyo_organization_id() iyo_see_doc = IYOSeeDocumentView(username=iyo_username, globalid=organization_id, uniqueid=u'Zero-Node order %s' % NodeOrder.create_human_readable_id(node_order_id), version=1, category=u'Terms and conditions', link=pdf_url, content_type=u'application/pdf', markdown_short_description=u'Terms and conditions for ordering a Zero-Node', markdown_full_description=u'Terms and conditions for ordering a Zero-Node') logging.debug('Creating IYO SEE document: %s', iyo_see_doc) iyo_see_doc = create_see_document(iyo_username, iyo_see_doc) attachment_name = u' - '.join([iyo_see_doc.uniqueid, iyo_see_doc.category]) def trans(): order = get_node_order(node_order_id) order.tos_iyo_see_id = iyo_see_doc.uniqueid order.put() deferred.defer(_create_quotation, app_user, node_order_id, pdf_url, attachment_name, _transactional=True) ndb.transaction(trans)
def make_session(userinfo, expiration_sec): """Creates new AuthOpenIDSession (and AuthOpenIDUser if needed) entities. Args: userinfo: user profile dict as returned by handle_authorization_code. expiration_sec: how long (in seconds) the session if allowed to live. Returns: AuthOpenIDSession already persisted in the datastore. """ now = utils.utcnow() # Refresh datastore entry for logged in user. user = AuthOpenIDUser( id=userinfo['sub'].encode('ascii'), last_session_ts=now, email=userinfo['email'], name=userinfo['name'], picture=userinfo['picture']) # Create a new session that expires at the same time when cookie signature # expires. ID is autogenerated by the datastore. session = AuthOpenIDSession( parent=user.key, created_ts=now, expiration_ts=now + datetime.timedelta(seconds=expiration_sec), email=user.email, name=user.name, picture=user.picture) ndb.transaction(lambda: ndb.put_multi([user, session])) assert session.key.integer_id() return session
def service_delete_provider(service_identity_user, provider_id, test_mode=False): def trans(): ps = get_payment_service(service_identity_user) if ps and ps.remove_provider(provider_id, test_mode): ps.put() ndb.transaction(trans)
def set_vote(quote_id, user_id, newvote, provider): if user_id is None: return if isinstance(user_id, unicode): auth_id = '%s:%s' % (provider, user_id) user = User.get_by_auth_id(auth_id) else: user = user_id email = user.email def txn(): quote = Post.get_by_id(quote_id) vote = Vote.get_by_id(id=email, parent=quote.key) if vote is None: vote = Vote(id=email, parent=quote.key) if vote.vote == newvote: return quote.votesum = quote.votesum - vote.vote + newvote vote.vote = newvote quote.rank = "%020d|%s" % (long(quote.created * DAY_SCALE + quote.votesum), quote.creation_order) quote.put() vote.put() memcache.set("vote|" + email + "|" + str(quote_id), vote.vote) ndb.transaction(txn)
def create(cls, args): """Create or update an invitation. Any given owner can have just invite per date.""" invite = Invitation.query(Invitation.date == args['when']).\ filter(Invitation.owner == args['owner']).get() if invite: invite.location = args['where'] invite.notes = args['notes'] invite.priority = args['priority'] updated = True else: invite = Invitation(date = args['when'].date(), time = args['when'].time(), owner = args['owner'], location = args['where'], notes = args['notes'], priority = args['priority'], parent = Invitation.dummy(), ) updated = False ndb.transaction(lambda: invite.put()) return updated, invite
def test_xaction(self): ent = MyModel() ent.put() def work(fail=False): ver = locks.Lock.get("mylock").ver entity = ent.key.get() entity.val += 2 entity.put() locks.Lock.incr("mylock") if fail: raise Exception("operation failed") ndb.transaction(work, xg=True) ent = ent.key.get() lock = locks.Lock.get("mylock") self.assertEqual(ent.val, 2) self.assertEqual(lock.ver, 1) try: ndb.transaction(lambda: work(fail=True), xg=True) except: pass ent = ent.key.get() lock = locks.Lock.get("mylock") self.assertEqual(ent.val, 2) # same as after success self.assertEqual(lock.ver, 1) # unchanged
def transfer_multi(cls, txs, transactional=None, xg=None, silent=None): """ Process multiple transactions as a single batch. With transactional set to True it's only possible to transfer funds from one source account to unlimited number of destination accounts. This limit can be expanded up to 5 source accounts by setting xg to True. Args: txs: List of dictionaries with transaction keyword arguments. transaction: Will be processed in a single transaction if True. xg: Enable cross-group transactions if True. silent: Returns: List of created transfer NDB objects. """ result = [] def _tx(): for kwargs in txs: if silent is None: result.append(cls.transfer_funds(**kwargs)) else: try: result.append(cls.transfer_funds(**kwargs)) except ValueError: pass if transactional: ndb.transaction(lambda: _tx(), xg=xg) else: _tx() return result
def testIsVotingAllowed_DisableHasFlaggedChecks(self): blockables = test_utils.CreateSantaBlockables(26) bundle = test_utils.CreateSantaBundle(bundle_binaries=blockables) # Flag one of the binaries. blockables[0].flagged = True blockables[0].put() with self.LoggedInUser(): # Ensure that the normal call succeeds in finding the flagged binary. allowed, reason = bundle.IsVotingAllowed() self.assertFalse(allowed) self.assertEqual( constants.VOTING_PROHIBITED_REASONS.FLAGGED_BINARY, reason) # In a transaction, the 26 searched blockables should exceed the allowed # limit of 25. with self.assertRaises(db.BadRequestError): ndb.transaction( lambda: bundle.IsVotingAllowed(enable_flagged_checks=True), xg=True) # With the checks disabled, IsVotingAllowed shouldn't raise an exception. def Test(): allowed, reason = bundle.IsVotingAllowed( enable_flagged_checks=False) self.assertTrue(allowed) self.assertIsNone(reason) ndb.transaction(Test, xg=True)
def updateAverageRating(review_key): """Helper function for updating the average rating of a product when new review(s) are added.""" def _tx(): review = review_key.get() product = review.product_key.get() if not review.rating_added: review.rating_added = True product.num_reviews += 1 product.avg_rating = (product.avg_rating + (review.rating - product.avg_rating)/float(product.num_reviews)) # signal that we need to reindex the doc with the new ratings info. product.needs_review_reindex = True ndb.put_multi([product, review]) if not config.BATCH_RATINGS_UPDATE: defer( models.Product.updateProdDocWithNewRating, product.key.id(), _transactional=True) return (product, review) try: # use an XG transaction in order to update both entities at once ndb.transaction(_tx, xg=True) except AttributeError: # swallow this error and log it; it's not recoverable. logging.exception('The function updateAverageRating failed. Either review ' + 'or product entity does not exist.')
def test_demonstrate_pattern(self): """ This just demonstrates typical usage """ # setup, pretend this was done before for i in range(0, 5): ent = MyModel(val=i) ent.put() ## pattern starts here ## # check the ver of particular lock before starting work verstart = locks.Lock.get("mylock").ver # now pretend like some other process changed the ver # after you started working locks.Lock.incr("mylock") item = MyModel.query(MyModel.val == 3).get() def work(): item.val = 100 item.put() vercheck = locks.Lock.get("mylock").ver if vercheck != verstart: # stop right here; don't process those items # try it again later, or raise an exception, or ... raise ndb.Rollback("versioning or sequence issue") ndb.transaction(work, retries=0, xg=True) ## pattern stops here ## # this just checks that the xaction was rolled back for item in MyModel.query(): # this is true because that work operation # never committed self.assertTrue(item.val < 100)
def post(self): if not referer_check(self.request): return uid = self.session.get('uid') if not uid: return ndb.transaction(lambda: unscribe_transaction(uid)) self.response.write('')
def record(cls, loser, winner, generation): def _tx(): id = "%d/%d" % (loser.id(), winner.id()) vote = cls.get_by_id(id) if not vote: vote = cls(id=id, loser=loser, winner=winner, generation=generation) vote.count += 1 vote.put() ndb.transaction(_tx)
def post(self): if not referer_check(self.request): return uid = self.session.get('uid') endpoint = self.request.get('endpoint') if not uid or not endpoint: return ndb.transaction(lambda: scribe_transaction(uid, endpoint)) self.response.write('')
def replicate_auth_db(): """Increments auth_db_rev by one. It is a signal that Auth DB should be replicated to Replicas. If called from inside a transaction, it inherits it and updates auth_db_rev only once (even if called multiple times during that transaction). Should only be called for services in Standalone or Primary modes. Will raise ValueError if called on Replica. When called for service in Standalone mode, will update auth_db_rev but won't kick any replication. For services in Primary mode will also initiate replication by calling callback set in 'configure_as_primary'. WARNING: This function relies on a valid transaction context. NDB hooks and asynchronous operations are known to be buggy in this regard: NDB hook for an async operation in a transaction may be called with a wrong context (main event loop context instead of transaction context). One way to work around that is to monkey patch NDB (as done here: https://goo.gl/1yASjL). Another is to not use hooks at all. There's no way to differentiate between sync and async modes of an NDB operation from inside a hook. And without a strict assert it's very easy to forget about "Do not use put_async" warning. For that reason _post_put_hook is NOT used and replicate_auth_db() should be called explicitly whenever relevant part of root_key() entity group is updated. """ def increment_revision_and_update_replicas(): """Does the actual job, called inside a transaction.""" # Update auth_db_rev. replication_state_key() is in same group as root_key. state = replication_state_key().get() if not state: primary_id = app_identity.get_application_id() if is_primary() else None state = AuthReplicationState( key=replication_state_key(), primary_id=primary_id, auth_db_rev=0) # Assert Primary or Standalone. Replicas can't increment auth db revision. if not is_primary() and state.primary_id: raise ValueError('Can\'t modify Auth DB on Replica') state.auth_db_rev += 1 state.modified_ts = utils.utcnow() state.put() # Only Primary does active replication. if is_primary(): _replication_callback(state) # If not in a transaction, start a new one. if not ndb.in_transaction(): ndb.transaction(increment_revision_and_update_replicas) return # If in a transaction, use transaction context to store "already did this" # flag. Note that each transaction retry gets its own new transaction context, # see ndb/context.py, 'transaction' tasklet, around line 982 (for SDK 1.9.6). ctx = ndb.get_context() if not getattr(ctx, '_auth_db_inc_called', False): increment_revision_and_update_replicas() ctx._auth_db_inc_called = True
def _cancel_quotation(order_id): def trans(): node_order = get_node_order(order_id) if node_order.odoo_sale_order_id: update_odoo_quotation(node_order.odoo_sale_order_id, {'state': QuotationState.CANCEL.value}) node_order.populate(status=NodeOrderStatus.CANCELED, cancel_time=now()) node_order.put() ndb.transaction(trans)
def get(self): bm = Bookmarks.get_by_id(int(self.request.get('bm'))) if users.get_current_user() == bm.user: def txn(): bm.url = self.request.get('url').encode('utf8') bm.title = self.request.get('title').encode('utf8') bm.comment = self.request.get('comment').encode('utf8') bm.put() ndb.transaction(txn) self.redirect('/')
def api_set_referral(params, user_detail): def trans(): code = params.get("code") if not code: raise ApiCallException(u'Unknown invitation code received') pp = ProfilePointer.get_by_user_code(code) if not pp: raise ApiCallException(u'Unknown invitation code received') username = get_iyo_username(user_detail) if username == pp.username: raise ApiCallException(u'You can\'t use your own invitation code') my_profile = TffProfile.create_key(username).get() # type: TffProfile if not my_profile: raise ApiCallException(u'We were unable to find your profile') if my_profile.referrer_user: raise ApiCallException(u'You already set your referrer') referrer_profile = TffProfile.create_key(pp.username).get() if not referrer_profile: raise ApiCallException( u'We were unable to find your referrer\'s profile') my_profile.referrer_user = referrer_profile.app_user my_profile.referrer_username = pp.username my_profile.put() deferred.defer(remove_user_from_role, user_detail, RogerthatRoles.PUBLIC, _transactional=True) deferred.defer(add_user_to_role, user_detail, RogerthatRoles.MEMBERS, _transactional=True) deferred.defer(remove_user_from_organization, username, Organization.PUBLIC, _transactional=True) deferred.defer(invite_user_to_organization, username, Organization.MEMBERS, _transactional=True) deferred.defer(store_referral_in_user_data, my_profile.key, _transactional=True) ndb.transaction(trans, xg=True) return { u'result': u'You successfully joined the ThreeFold community. Welcome aboard!' }
def post(self): dom = parseString(self.request.body) wsRoot = dom.getElementsByTagName('workspace').item(0) wsName = wsRoot.getAttribute('name') try: nextNoteNum = int(wsRoot.getAttribute('nextNoteNum')) except: nextNoteNum = 0 notesJsonArray = [] nlNotes = wsRoot.getElementsByTagName('note') for i in range(nlNotes.length): node = nlNotes.item(i) note = {} for j in range(node.attributes.length): attr = node.attributes.item(j) if attr.name in models.Notes.DBKEYS: # only use valid attributes note[models.Notes.DB2JS[attr.name]] = attr.nodeValue note['text'] = node.firstChild.nodeValue.strip() notesJsonArray.append(note) def txn(): nowtime = datetime.datetime.now().replace(microsecond=0) workspace = models.Workspace.get_by_wsName(wsName) if not workspace: workspace = models.Workspace.create(wsName, nextNoteNum, nowtime) workspace.put() workspace.time = nowtime workspace.nextNoteNum = nextNoteNum workspace.put() notes = models.Notes(workspaceKey=workspace.key, time=nowtime, notesJsonArray=notesJsonArray) notes.put() for i in range(5): try: ndb.transaction(txn, xg=True) break except db.TransactionFailedError: time.sleep(1.1) origin = self.request.headers.get('Origin', '') logging.info(origin) if origin in ('http://localhost:8080', 'http://www.aypwip.org', 'https://www.aypwip.org'): self.response.headers['Access-Control-Allow-Origin'] = origin self.response.content_type = 'text/xml' self.response.write( '<return><status value="ok" update="%s"/></return>' % '2017-11-01 12:12:12')
def testInTxn(self): def AssertInTxn(): self.assertTrue(ndb.in_transaction()) def RunAssert(): fut = utils.GetNoOpFuture() fut.add_callback(AssertInTxn) fut.add_immediate_callback(AssertInTxn) fut.get_result() ndb.transaction(RunAssert)
def setUp(self): super(TestOutput, self).setUp() request = task_request.make_request(_gen_request(), True) result_summary = task_result.new_result_summary(request) result_summary.modified_ts = utils.utcnow() ndb.transaction(result_summary.put) self.run_result = task_result.new_run_result(request, 1, "localhost", "abc", {}) self.run_result.modified_ts = utils.utcnow() result_summary.set_from_run_result(self.run_result, request) ndb.transaction(lambda: ndb.put_multi((result_summary, self.run_result))) self.run_result = self.run_result.key.get()
def delicious(tag, comment, user): link = tag.a bm = Bookmarks() def txn(): bm.original = link['href'] bm.title = link.text bm.comment = comment bm.user = user bm.put() ndb.transaction(txn) deferred.defer(main_parser, bm.key, _target="worker", _queue="parser")
def setUp(self): super(TestOutput, self).setUp() request = task_request.make_request(_gen_request_data()) result_summary = task_result.new_result_summary(request) result_summary.modified_ts = utils.utcnow() ndb.transaction(result_summary.put) self.run_result = task_result.new_run_result(request, 1, 'localhost', 'abc') self.run_result.modified_ts = utils.utcnow() result_summary.set_from_run_result(self.run_result, request) ndb.transaction(lambda: ndb.put_multi((result_summary, self.run_result))) self.run_result = self.run_result.key.get()
def get(self): old = Bookmarks.get_by_id(int(self.request.get('bm'))) bm = Bookmarks() def txn(): bm.original = old.original bm.title = old.title bm.comment = old.comment bm.user = users.get_current_user() bm.put() ndb.transaction(txn) deferred.defer(main_parser, bm.key, _queue="parser")
def get(self): bm = Bookmarks() def txn(): bm.original = self.request.get('url') bm.url = self.request.get('url') bm.title = self.request.get('title') bm.comment = self.request.get('comment') bm.user = users.User(str(self.request.get('user'))) bm.put() ndb.transaction(txn) main_parser(bm.key) self.redirect('/')
def flushcache(): # conditionally flush memcache if this is loaded # due to module deployment _cachestate = cachestate.get() if _cachestate.cleartime != gaeutils.App.deploy_time: logging.debug('flushing cache. last cleared %i and curr deploy time %i' % ( _cachestate.cleartime, gaeutils.App.deploy_time)) def txn(): _cachestate.cleartime = gaeutils.App.deploy_time _cachestate.put() if memcache.flush_all(): ndb.transaction(txn)
def post(self): if not referer_check(self.request): return uid = self.session.get('uid') if not uid: return pid = self.request.get('pid') focuced = False if self.request.get('v'): focuced = True ndb.transaction(lambda: change_focus_transaction(uid, pid, focuced)) self.response.write('')
def sendTestEmail(self, message_entity): """Send the test emails to the requested address. Args: messages_entity: Messages entity containing the messages to be emailed. """ assert access_checker.isSet(self.request_data.program) test_email_addr = self.cleaned_data.get('test_email', None) if not test_email_addr: return self.request_data.redirect.program() test_org_key = ndb.Key( self.request_data.models.org_model._get_kind(), '/'.join([self.request_data.program.key().name(), TEST_ORG_ID])) org_app_context = { 'url': links.ABSOLUTE_LINKER.organization( test_org_key, urls.UrlNames.ORG_PROFILE_EDIT), 'org': TEST_ORG_NAME, } proposal_context = { 'proposal_title': TEST_PROPOSAL_TITLE, 'org_entity': TEST_ORG_ENTITY, } mail_txns = [] if message_entity.accepted_orgs_msg: mail_txns.append(self.getSendMailFromTemplateStringTxn( test_email_addr, notifications.DEF_ACCEPTED_ORG % (org_app_context), message_entity.accepted_orgs_msg, org_app_context)) if message_entity.rejected_orgs_msg: mail_txns.append(self.getSendMailFromTemplateStringTxn( test_email_addr, notifications.DEF_REJECTED_ORG % (org_app_context), message_entity.rejected_orgs_msg, org_app_context)) if message_entity.accepted_students_msg: mail_txns.append(self.getSendMailFromTemplateStringTxn( test_email_addr, 'Congratulations!', message_entity.accepted_students_msg, proposal_context)) if message_entity.rejected_students_msg: mail_txns.append(self.getSendMailFromTemplateStringTxn( test_email_addr, 'Thank you for applying to %s' % (self.request_data.program.name), message_entity.rejected_students_msg, proposal_context)) for mail_txn in mail_txns: ndb.transaction(mail_txn)
def reduceProcess(data_id, entities): # TODO: (Aruna) Fix these import from melange.logic import cached_list from melange.utils import lists ctx = context.get() params = ctx.mapreduce_spec.mapper.params list_id = params['list_id'] ndb.transaction(lambda: cached_list.setCacheItems( data_id, map(json.loads, entities), lists.getList(list_id).valid_period))
def test_new_result_summary(self): request = task_request.make_request(_gen_request(), True) actual = task_result.new_result_summary(request) expected = { 'abandoned_ts': None, 'bot_dimensions': None, 'bot_id': None, 'bot_version': None, 'children_task_ids': [], 'completed_ts': None, 'costs_usd': [], 'cost_saved_usd': None, 'created_ts': self.now, 'deduped_from': None, 'duration': None, 'exit_code': None, 'failure': False, 'id': '1d69b9f088008810', 'internal_failure': False, 'modified_ts': None, 'name': u'Request name', 'outputs_ref': None, 'properties_hash': None, 'server_versions': [], 'started_ts': None, 'state': task_result.State.PENDING, 'tags': [ u'pool:default', u'priority:50', u'tag:1', u'user:Jesus', ], 'try_number': None, 'user': u'Jesus', } self.assertEqual(expected, actual.to_dict()) self.assertEqual(50, actual.request.priority) self.assertEqual(True, actual.can_be_canceled) actual.state = task_result.State.RUNNING self.assertEqual(False, actual.can_be_canceled) actual.children_task_ids = [ '1d69ba3ea8008810', '3d69ba3ea8008810', '2d69ba3ea8008810', ] actual.modified_ts = utils.utcnow() ndb.transaction(actual.put) expected = [ u'1d69ba3ea8008810', u'2d69ba3ea8008810', u'3d69ba3ea8008810' ] self.assertEqual(expected, actual.key.get().children_task_ids)
def _update_transaction_status(transaction_id, status): def trans(): transaction = PayconiqTransaction.create_key(transaction_id).get() if transaction.status != PayconiqTransaction.STATUS_PENDING and transaction.status != status: return transaction.status = status transaction.put() deferred.defer(send_update_payment_status_request_to_user, transaction.app_user, transaction_id, status, _transactional=True, _queue=FAST_QUEUE) ndb.transaction(trans)
def reduceProcess(data_id, entities): # TODO: (Aruna) Fix these import from melange.logic import cached_list from melange.utils import lists ctx = context.get() params = ctx.mapreduce_spec.mapper.params list_id = params['list_id'] ndb.transaction( lambda: cached_list.setCacheItems(data_id, map(json.loads, entities), lists.getList(list_id).valid_period))
def test_set_from_run_result(self): request = task_request.make_request(_gen_request(), True) result_summary = task_result.new_result_summary(request) run_result = task_result.new_run_result(request, 1, 'localhost', 'abc', {}) self.assertTrue(result_summary.need_update_from_run_result(run_result)) result_summary.modified_ts = utils.utcnow() run_result.modified_ts = utils.utcnow() ndb.transaction(lambda: ndb.put_multi((result_summary, run_result))) self.assertTrue(result_summary.need_update_from_run_result(run_result)) result_summary.set_from_run_result(run_result, request) ndb.transaction(lambda: ndb.put_multi([result_summary])) self.assertFalse(result_summary.need_update_from_run_result(run_result))
def test_set_from_run_result(self): request = mkreq(_gen_request()) result_summary = task_result.new_result_summary(request) run_result = task_result.new_run_result(request, 1, 'localhost', 'abc', {}) self.assertTrue(result_summary.need_update_from_run_result(run_result)) result_summary.modified_ts = utils.utcnow() run_result.modified_ts = utils.utcnow() ndb.transaction(lambda: ndb.put_multi((result_summary, run_result))) self.assertTrue(result_summary.need_update_from_run_result(run_result)) result_summary.set_from_run_result(run_result, request) ndb.transaction(lambda: ndb.put_multi([result_summary])) self.assertFalse(result_summary.need_update_from_run_result(run_result))
def test_run_result_timeout(self): request = task_request.make_request(_gen_request_data()) result_summary = task_result.new_result_summary(request) result_summary.modified_ts = utils.utcnow() ndb.transaction(result_summary.put) run_result = task_result.new_run_result(request, 1, 'localhost', 'abc') run_result.state = task_result.State.TIMED_OUT run_result.completed_ts = utils.utcnow() run_result.modified_ts = utils.utcnow() result_summary.set_from_run_result(run_result, request) ndb.transaction(lambda: ndb.put_multi((run_result, result_summary))) run_result = run_result.key.get() result_summary = result_summary.key.get() self.assertEqual(True, run_result.failure) self.assertEqual(True, result_summary.failure)
def test_run_result_timeout(self): request = task_request.make_request(_gen_request(), True) result_summary = task_result.new_result_summary(request) result_summary.modified_ts = utils.utcnow() ndb.transaction(result_summary.put) run_result = task_result.new_run_result(request, 1, 'localhost', 'abc', {}) run_result.state = task_result.State.TIMED_OUT run_result.completed_ts = utils.utcnow() run_result.modified_ts = utils.utcnow() result_summary.set_from_run_result(run_result, request) ndb.transaction(lambda: ndb.put_multi((run_result, result_summary))) run_result = run_result.key.get() result_summary = result_summary.key.get() self.assertEqual(True, run_result.failure) self.assertEqual(True, result_summary.failure)
def get(self): logging.info("REBOOT") send_to = set() to_delete = set() msg = json.dumps({ "reboot": True }) for e in events.query(): for l in e.listeners: if l not in send_to: sent_to.add(l) to_delete.add(e.key) def delete(): ndb.delete_multi(to_delete) ndb.transaction(delete) for l in send_to: deferred.defer(channel.send_message, str(l), msg)
def test_new_result_summary(self): request = task_request.make_request(_gen_request(), True) actual = task_result.new_result_summary(request) expected = { 'abandoned_ts': None, 'bot_dimensions': None, 'bot_id': None, 'bot_version': None, 'children_task_ids': [], 'completed_ts': None, 'costs_usd': [], 'cost_saved_usd': None, 'created_ts': self.now, 'deduped_from': None, 'duration': None, 'exit_code': None, 'failure': False, 'id': '1d69b9f088008810', 'internal_failure': False, 'modified_ts': None, 'name': u'Request name', 'outputs_ref': None, 'properties_hash': None, 'server_versions': [], 'started_ts': None, 'state': task_result.State.PENDING, 'tags': [ u'pool:default', u'priority:50', u'tag:1', u'user:Jesus', ], 'try_number': None, 'user': u'Jesus', } self.assertEqual(expected, actual.to_dict()) self.assertEqual(50, actual.request.priority) self.assertEqual(True, actual.can_be_canceled) actual.state = task_result.State.RUNNING self.assertEqual(False, actual.can_be_canceled) actual.children_task_ids = [ '1d69ba3ea8008810', '3d69ba3ea8008810', '2d69ba3ea8008810', ] actual.modified_ts = utils.utcnow() ndb.transaction(actual.put) expected = [u'1d69ba3ea8008810', u'2d69ba3ea8008810', u'3d69ba3ea8008810'] self.assertEqual(expected, actual.key.get().children_task_ids)