def make_session(userinfo, expiration_sec):
"""Creates new AuthOpenIDSession (and AuthOpenIDUser if needed) entities.
Args:
userinfo: user profile dict as returned by handle_authorization_code.
expiration_sec: how long (in seconds) the session if allowed to live.
Returns:
AuthOpenIDSession already persisted in the datastore.
"""
now = utils.utcnow()
# Refresh datastore entry for logged in user.
user = AuthOpenIDUser(
id=userinfo["sub"].encode("ascii"),
last_session_ts=now,
email=userinfo["email"],
name=userinfo["name"],
picture=userinfo["picture"],
)
# Create a new session that expires at the same time when cookie signature
# expires. ID is autogenerated by the datastore.
session = AuthOpenIDSession(
parent=user.key,
created_ts=now,
expiration_ts=now + datetime.timedelta(seconds=expiration_sec),
email=user.email,
name=user.name,
picture=user.picture,
)
ndb.transaction(lambda: ndb.put_multi([user, session]))
assert session.key.integer_id()
return session
def test_get_remove_identifiers(self):
"""
Tests the get and remove by identifier functions
"""
tree = MultiBTree2.create("tree", 3)
tree.update((1, str(x), str(x)) for x in range(30))
for x in range(30):
self.assertEqual((1, str(x), str(x)),
tree.get_by_identifier(str(x)))
for x in range(30):
tree.remove_by_identifier(str(x))
self.assertEqual(None, tree.get_by_identifier(str(x)))
self.assertEqual(tree.tree_size(), 0)
item = (10, "10", "id")
item2 = (11, "11", "id2")
def txn():
tree.update([item, item2])
self.assertEqual(10 in tree, True)
self.assertEqual(11 in tree, True)
self.assertEqual(tree.get_by_identifier("id"), item)
self.assertEqual(tree.get_by_identifier("id2"), item2)
tree.remove_by_identifier("id")
self.assertEqual(tree.get_by_identifier("id"), None)
ndb.transaction(txn)
self.assertEqual(tree.tree_size(), 1)
self.assertEqual(tree.get_by_identifier("id"), None)
self.assertEqual(tree.get_by_identifier("id2"), item2)
self.validate_indices(tree)
def post(self):
from libs.feedparser import parse
user = users.get_current_user()
url = self.request.get('url')
p = parse(str(url))
try:
d = p['items'][0]
except IndexError:
pass
if user:
q = Feeds.query(Feeds.user == user, Feeds.url == url)
if q.get() is None:
feed = Feeds()
def txn():
feed.blog = p.feed.title
feed.root = p.feed.link
feed.user = user
feed.feed = url
feed.url = d.link
feed.put()
ndb.transaction(txn)
deferred.defer(utils.new_bm, d, feed.key, _queue="admin")
self.redirect(self.request.referer)
else:
self.redirect('/')
def _order_node_iyo_see(app_user,
node_order_id,
pdf_url,
pdf_size,
create_quotation=True):
iyo_username = get_iyo_username(app_user)
doc_id = u'Zero-Node order %s' % NodeOrder.create_human_readable_id(
node_order_id)
category = u'Terms and conditions'
content_type = u'application/pdf'
description = u'Terms and conditions for ordering a Zero-Node'
create_see_document(doc_id, category, description, iyo_username, pdf_url,
content_type)
attachment_name = u' - '.join([doc_id, category])
def trans():
order = get_node_order(node_order_id)
order.tos_iyo_see_id = doc_id
order.put()
if create_quotation:
deferred.defer(_create_quotation,
app_user,
node_order_id,
pdf_url,
attachment_name,
pdf_size,
_transactional=True)
ndb.transaction(trans)
def save(self, must_create=False):
"""Create and save a Session object using db.run_in_transaction, with
key_name = session_key, raising CreateError if
unsuccessful.
"""
if must_create:
s = self.get_ndb_session_key().get()
if s:
raise CreateError()
session_data = self._get_session(no_load=must_create)
#ed = self.get_expiry_date()
#print datetime.datetime.utcoffset(ed)
def txn():
s = Session(id=self._get_or_create_session_key(),
session_key=self.session_key,
session_data=self.encode(session_data),
expire_date=self.get_expiry_date())
s.put()
# This is tricky and probably needs some sanity checking, because
# TransactionFailedError can be raised, but the transaction can still
# go on to be committed to the datastore. As far as I can see there's
# no way to manually roll it back at that point. No idea how to test
# this either.
try:
ndb.transaction(txn)
except (ndb.Rollback):
raise CreateError()
def test_new_result_summary(self):
request = _gen_request()
actual = task_result.new_result_summary(request)
actual.modified_ts = self.now
# Trigger _pre_put_hook().
actual.put()
expected = self._gen_summary(modified_ts=self.now)
self.assertEqual(expected, actual.to_dict())
self.assertEqual(50, actual.request.priority)
self.assertEqual(True, actual.can_be_canceled)
actual.state = task_result.State.RUNNING
self.assertEqual(True, actual.can_be_canceled)
actual.state = task_result.State.TIMED_OUT
actual.duration = 0.1
self.assertEqual(False, actual.can_be_canceled)
actual.children_task_ids = [
'1d69ba3ea8008810',
'3d69ba3ea8008810',
'2d69ba3ea8008810',
]
actual.modified_ts = utils.utcnow()
ndb.transaction(actual.put)
expected = [
u'1d69ba3ea8008810', u'2d69ba3ea8008810', u'3d69ba3ea8008810'
]
self.assertEqual(expected, actual.key.get().children_task_ids)
def updateAverageRating(review_key):
"""Helper function for updating the average rating of a product when new
review(s) are added."""
def _tx():
review = review_key.get()
v = review.video_key.get()
if not review.rating_added:
review.rating_added = True
v.num_reviews += 1
v.avg_rating = (
v.avg_rating +
(review.rating - v.avg_rating) / float(v.num_reviews))
# signal that we need to reindex the doc with the new ratings info.
v.needs_review_reindex = True
ndb.put_multi([v, review])
# We need to update the ratings associated document at some point as well.
# If the app is configured to have BATCH_RATINGS_UPDATE set to True, don't
# do this re-indexing now. (Instead, all the out-of-date documents can be
# be later handled in batch -- see cron.yaml). If BATCH_RATINGS_UPDATE is
# False, go ahead and reindex now in a transational task.
if not config.BATCH_RATINGS_UPDATE:
defer(models.Video.updateVideoDocWithNewRating,
v.key.id(),
_transactional=True)
return (v, review)
try:
# use an XG transaction in order to update both entities at once
ndb.transaction(_tx, xg=True)
except AttributeError:
# swallow this error and log it; it's not recoverable.
logging.exception(
'The function updateAverageRating failed. Either review ' +
'or product entity does not exist.')
def updateProdDocsWithNewRating(cls, pkeys):
"""Given a list of product entity keys, check each entity to see if it is
marked as needing a document re-index. This flag is set when a new review
is created for that product, and config.BATCH_RATINGS_UPDATE = True.
Generate the modified docs as needed and batch re-index them."""
doclist = []
def _tx(pid):
prod = cls.get_by_id(pid)
if prod and prod.needs_review_reindex:
# update the associated document with the new ratings info
# and reindex
modified_doc = docs.Product.updateRatingInDoc(
prod.doc_id, prod.avg_rating)
if modified_doc:
doclist.append(modified_doc)
prod.needs_review_reindex = False
prod.put()
for pkey in pkeys:
ndb.transaction(lambda: _tx(pkey.id()))
# reindex all modified docs in batch
docs.Product.add(doclist)
def investment_agreement_signed_by_admin(status, form_result, answer_id, member, message_key, tag, received_timestamp,
acked_timestamp, parent_message_key, result_key, service_identity,
user_details):
tag_dict = json.loads(tag)
def trans():
agreement = InvestmentAgreement.create_key(tag_dict['agreement_id']).get() # type: InvestmentAgreement
if answer_id != FormTO.POSITIVE:
logging.info('Investment agreement sign aborted')
return
if agreement.status == InvestmentAgreement.STATUS_PAID:
logging.warn('Ignoring request to set InvestmentAgreement %s as paid because it is already paid',
agreement.id)
return
agreement.status = InvestmentAgreement.STATUS_PAID
agreement.paid_time = now()
agreement.put()
user_email, app_id, = get_app_user_tuple(agreement.app_user)
deferred.defer(transfer_genesis_coins_to_user, agreement.app_user, TokenType.I,
long(agreement.token_count_float * 100), _transactional=True)
deferred.defer(update_investor_progress, user_email.email(), app_id, INVESTMENT_TODO_MAPPING[agreement.status],
_transactional=True)
deferred.defer(_send_tokens_assigned_message, agreement.app_user, _transactional=True)
ndb.transaction(trans)
def _update_transaction_with_payconiq_info(transaction_id, payconic_transaction_id):
def trans():
transaction = PayconiqTransaction.create_key(transaction_id).get()
transaction.payconic_transaction_id = payconic_transaction_id
transaction.put()
ndb.transaction(trans)
def updateAverageRating(review_key):
"""Helper function for updating the average rating of a product when new
review(s) are added."""
def _tx():
review = review_key.get()
product = review.product_key.get()
if not review.rating_added:
review.rating_added = True
product.num_reviews += 1
product.avg_rating = (product.avg_rating +
(review.rating - product.avg_rating)/float(product.num_reviews))
# signal that we need to reindex the doc with the new ratings info.
product.needs_review_reindex = True
ndb.put_multi([product, review])
# We need to update the ratings associated document at some point as well.
# If the app is configured to have BATCH_RATINGS_UPDATE set to True, don't
# do this re-indexing now. (Instead, all the out-of-date documents can be
# be later handled in batch -- see cron.yaml). If BATCH_RATINGS_UPDATE is
# False, go ahead and reindex now in a transational task.
if not config.BATCH_RATINGS_UPDATE:
defer(
models.Product.updateProdDocWithNewRating,
product.key.id(), _transactional=True)
return (product, review)
try:
# use an XG transaction in order to update both entities at once
ndb.transaction(_tx, xg=True)
except AttributeError:
# swallow this error and log it; it's not recoverable.
logging.exception('The function updateAverageRating failed. Either review '
+ 'or product entity does not exist.')
def test_set_from_run_result_two_server_versions(self):
request = _gen_request()
result_summary = task_result.new_result_summary(request)
to_run = task_to_run.new_task_to_run(request, 1, 0)
run_result = task_result.new_run_result(request, to_run, 'localhost',
'abc', {})
run_result.started_ts = utils.utcnow()
self.assertTrue(result_summary.need_update_from_run_result(run_result))
result_summary.modified_ts = utils.utcnow()
run_result.modified_ts = utils.utcnow()
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result)))
self.assertTrue(result_summary.need_update_from_run_result(run_result))
ndb.transaction(
lambda: result_summary.set_from_run_result(run_result, request))
ndb.transaction(lambda: ndb.put_multi([result_summary]))
run_result.signal_server_version('new-version')
run_result.modified_ts = utils.utcnow()
ndb.transaction(
lambda: result_summary.set_from_run_result(run_result, request))
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result)))
self.assertEqual(['v1a', 'new-version'],
run_result.key.get().server_versions)
self.assertEqual(['v1a', 'new-version'],
result_summary.key.get().server_versions)
def save(self, must_create=False):
"""Create and save a Session object using db.run_in_transaction, with
key_name = session_key, raising CreateError if
unsuccessful.
"""
if must_create:
s = self.get_ndb_session_key().get()
if s:
raise CreateError()
session_data = self._get_session(no_load=must_create)
#ed = self.get_expiry_date()
#print datetime.datetime.utcoffset(ed)
def txn():
s = Session(
id=self._get_or_create_session_key(),
session_key=self.session_key,
session_data=self.encode(session_data),
expire_date=self.get_expiry_date()
)
s.put()
# This is tricky and probably needs some sanity checking, because
# TransactionFailedError can be raised, but the transaction can still
# go on to be committed to the datastore. As far as I can see there's
# no way to manually roll it back at that point. No idea how to test
# this either.
try:
ndb.transaction(txn)
except (ndb.Rollback):
raise CreateError()
def backfill_tags(entity):
# Already handled?
if entity.tags:
return
# TaskRequest is immutable, can be fetched outside the transaction.
task_request = entity.request_key.get(use_cache=False, use_memcache=False)
if not task_request or not task_request.tags:
return
# Fast path for old entries: do not use transaction, assumes old entities are
# not being concurrently modified outside of this job.
if entity.created_ts and entity.created_ts < OLD_TASKS_CUTOFF:
entity.tags = task_request.tags
yield operation.db.Put(entity)
return
# For recent entries be careful and use transaction.
def fix_task_result_summary():
task_result_summary = entity.key.get()
if task_result_summary and not task_result_summary.tags:
task_result_summary.tags = task_request.tags
task_result_summary.put()
ndb.transaction(fix_task_result_summary, use_cache=False, use_memcache=False)
def set_vote(quote_id, user_id, newvote, provider):
if user_id is None:
return
if isinstance(user_id, unicode):
auth_id = '%s:%s' % (provider, user_id)
user = User.get_by_auth_id(auth_id)
else:
user = user_id
email = user.email
def txn():
quote = Post.get_by_id(quote_id)
vote = Vote.get_by_id(id=email, parent=quote.key)
if vote is None:
vote = Vote(id=email, parent=quote.key)
if vote.vote == newvote:
return
quote.votesum = quote.votesum - vote.vote + newvote
vote.vote = newvote
quote.rank = "%020d|%s" % (
long(quote.created * DAY_SCALE + quote.votesum),
quote.creation_order
)
quote.put()
vote.put()
memcache.set("vote|" + email + "|" + str(quote_id), vote.vote)
ndb.transaction(txn)
def _order_node_iyo_see(app_user, node_order_id, pdf_url):
iyo_username = get_iyo_username(app_user)
organization_id = get_iyo_organization_id()
iyo_see_doc = IYOSeeDocumentView(username=iyo_username,
globalid=organization_id,
uniqueid=u'Zero-Node order %s' % NodeOrder.create_human_readable_id(node_order_id),
version=1,
category=u'Terms and conditions',
link=pdf_url,
content_type=u'application/pdf',
markdown_short_description=u'Terms and conditions for ordering a Zero-Node',
markdown_full_description=u'Terms and conditions for ordering a Zero-Node')
logging.debug('Creating IYO SEE document: %s', iyo_see_doc)
iyo_see_doc = create_see_document(iyo_username, iyo_see_doc)
attachment_name = u' - '.join([iyo_see_doc.uniqueid, iyo_see_doc.category])
def trans():
order = get_node_order(node_order_id)
order.tos_iyo_see_id = iyo_see_doc.uniqueid
order.put()
deferred.defer(_create_quotation, app_user, node_order_id, pdf_url, attachment_name,
_transactional=True)
ndb.transaction(trans)
def test_get_remove_identifiers(self):
"""
Tests the get and remove by identifier functions
"""
tree = MultiBTree2.create("tree", 3)
tree.update((1, str(x), str(x)) for x in range(30))
for x in range(30):
self.assertEqual((1, str(x), str(x)),
tree.get_by_identifier(str(x)))
for x in range(30):
tree.remove_by_identifier(str(x))
self.assertEqual(None, tree.get_by_identifier(str(x)))
self.assertEqual(tree.tree_size(), 0)
item = (10, "10", "id")
item2 = (11, "11", "id2")
def txn():
tree.update([item, item2])
self.assertEqual(10 in tree, True)
self.assertEqual(11 in tree, True)
self.assertEqual(tree.get_by_identifier("id"), item)
self.assertEqual(tree.get_by_identifier("id2"), item2)
tree.remove_by_identifier("id")
self.assertEqual(tree.get_by_identifier("id"), None)
ndb.transaction(txn)
self.assertEqual(tree.tree_size(), 1)
self.assertEqual(tree.get_by_identifier("id"), None)
self.assertEqual(tree.get_by_identifier("id2"), item2)
self.validate_indices(tree)
def make_session(userinfo, expiration_sec):
"""Creates new AuthOpenIDSession (and AuthOpenIDUser if needed) entities.
Args:
userinfo: user profile dict as returned by handle_authorization_code.
expiration_sec: how long (in seconds) the session if allowed to live.
Returns:
AuthOpenIDSession already persisted in the datastore.
"""
now = utils.utcnow()
# Refresh datastore entry for logged in user.
user = AuthOpenIDUser(
id=userinfo['sub'].encode('ascii'),
last_session_ts=now,
email=userinfo['email'],
name=userinfo['name'],
picture=userinfo['picture'])
# Create a new session that expires at the same time when cookie signature
# expires. ID is autogenerated by the datastore.
session = AuthOpenIDSession(
parent=user.key,
created_ts=now,
expiration_ts=now + datetime.timedelta(seconds=expiration_sec),
email=user.email,
name=user.name,
picture=user.picture)
ndb.transaction(lambda: ndb.put_multi([user, session]))
assert session.key.integer_id()
return session
def service_delete_provider(service_identity_user, provider_id, test_mode=False):
def trans():
ps = get_payment_service(service_identity_user)
if ps and ps.remove_provider(provider_id, test_mode):
ps.put()
ndb.transaction(trans)
def set_vote(quote_id, user_id, newvote, provider):
if user_id is None:
return
if isinstance(user_id, unicode):
auth_id = '%s:%s' % (provider, user_id)
user = User.get_by_auth_id(auth_id)
else:
user = user_id
email = user.email
def txn():
quote = Post.get_by_id(quote_id)
vote = Vote.get_by_id(id=email, parent=quote.key)
if vote is None:
vote = Vote(id=email, parent=quote.key)
if vote.vote == newvote:
return
quote.votesum = quote.votesum - vote.vote + newvote
vote.vote = newvote
quote.rank = "%020d|%s" % (long(quote.created * DAY_SCALE +
quote.votesum), quote.creation_order)
quote.put()
vote.put()
memcache.set("vote|" + email + "|" + str(quote_id), vote.vote)
ndb.transaction(txn)
def create(cls, args):
"""Create or update an invitation.
Any given owner can have just invite per date."""
invite = Invitation.query(Invitation.date == args['when']).\
filter(Invitation.owner == args['owner']).get()
if invite:
invite.location = args['where']
invite.notes = args['notes']
invite.priority = args['priority']
updated = True
else:
invite = Invitation(date = args['when'].date(),
time = args['when'].time(),
owner = args['owner'],
location = args['where'],
notes = args['notes'],
priority = args['priority'],
parent = Invitation.dummy(),
)
updated = False
ndb.transaction(lambda: invite.put())
return updated, invite
def test_xaction(self):
ent = MyModel()
ent.put()
def work(fail=False):
ver = locks.Lock.get("mylock").ver
entity = ent.key.get()
entity.val += 2
entity.put()
locks.Lock.incr("mylock")
if fail:
raise Exception("operation failed")
ndb.transaction(work, xg=True)
ent = ent.key.get()
lock = locks.Lock.get("mylock")
self.assertEqual(ent.val, 2)
self.assertEqual(lock.ver, 1)
try:
ndb.transaction(lambda: work(fail=True), xg=True)
except:
pass
ent = ent.key.get()
lock = locks.Lock.get("mylock")
self.assertEqual(ent.val, 2) # same as after success
self.assertEqual(lock.ver, 1) # unchanged
def transfer_multi(cls, txs, transactional=None, xg=None, silent=None):
""" Process multiple transactions as a single batch.
With transactional set to True it's only possible to transfer funds from
one source account to unlimited number of destination accounts.
This limit can be expanded up to 5 source accounts by setting xg to True.
Args:
txs: List of dictionaries with transaction keyword arguments.
transaction: Will be processed in a single transaction if True.
xg: Enable cross-group transactions if True.
silent:
Returns:
List of created transfer NDB objects.
"""
result = []
def _tx():
for kwargs in txs:
if silent is None:
result.append(cls.transfer_funds(**kwargs))
else:
try:
result.append(cls.transfer_funds(**kwargs))
except ValueError:
pass
if transactional:
ndb.transaction(lambda: _tx(), xg=xg)
else:
_tx()
return result
def testIsVotingAllowed_DisableHasFlaggedChecks(self):
blockables = test_utils.CreateSantaBlockables(26)
bundle = test_utils.CreateSantaBundle(bundle_binaries=blockables)
# Flag one of the binaries.
blockables[0].flagged = True
blockables[0].put()
with self.LoggedInUser():
# Ensure that the normal call succeeds in finding the flagged binary.
allowed, reason = bundle.IsVotingAllowed()
self.assertFalse(allowed)
self.assertEqual(
constants.VOTING_PROHIBITED_REASONS.FLAGGED_BINARY, reason)
# In a transaction, the 26 searched blockables should exceed the allowed
# limit of 25.
with self.assertRaises(db.BadRequestError):
ndb.transaction(
lambda: bundle.IsVotingAllowed(enable_flagged_checks=True),
xg=True)
# With the checks disabled, IsVotingAllowed shouldn't raise an exception.
def Test():
allowed, reason = bundle.IsVotingAllowed(
enable_flagged_checks=False)
self.assertTrue(allowed)
self.assertIsNone(reason)
ndb.transaction(Test, xg=True)
def updateAverageRating(review_key):
"""Helper function for updating the average rating of a product when new
review(s) are added."""
def _tx():
review = review_key.get()
product = review.product_key.get()
if not review.rating_added:
review.rating_added = True
product.num_reviews += 1
product.avg_rating = (product.avg_rating +
(review.rating - product.avg_rating)/float(product.num_reviews))
# signal that we need to reindex the doc with the new ratings info.
product.needs_review_reindex = True
ndb.put_multi([product, review])
if not config.BATCH_RATINGS_UPDATE:
defer(
models.Product.updateProdDocWithNewRating,
product.key.id(), _transactional=True)
return (product, review)
try:
# use an XG transaction in order to update both entities at once
ndb.transaction(_tx, xg=True)
except AttributeError:
# swallow this error and log it; it's not recoverable.
logging.exception('The function updateAverageRating failed. Either review '
+ 'or product entity does not exist.')
def backfill_tags(entity):
# Already handled?
if entity.tags:
return
# TaskRequest is immutable, can be fetched outside the transaction.
task_request = entity.request_key.get(use_cache=False, use_memcache=False)
if not task_request or not task_request.tags:
return
# Fast path for old entries: do not use transaction, assumes old entities are
# not being concurrently modified outside of this job.
if entity.created_ts and entity.created_ts < OLD_TASKS_CUTOFF:
entity.tags = task_request.tags
yield operation.db.Put(entity)
return
# For recent entries be careful and use transaction.
def fix_task_result_summary():
task_result_summary = entity.key.get()
if task_result_summary and not task_result_summary.tags:
task_result_summary.tags = task_request.tags
task_result_summary.put()
ndb.transaction(fix_task_result_summary,
use_cache=False,
use_memcache=False)
def create(cls, args):
"""Create or update an invitation.
Any given owner can have just invite per date."""
invite = Invitation.query(Invitation.date == args['when']).\
filter(Invitation.owner == args['owner']).get()
if invite:
invite.location = args['where']
invite.notes = args['notes']
invite.priority = args['priority']
updated = True
else:
invite = Invitation(date = args['when'].date(),
time = args['when'].time(),
owner = args['owner'],
location = args['where'],
notes = args['notes'],
priority = args['priority'],
parent = Invitation.dummy(),
)
updated = False
ndb.transaction(lambda: invite.put())
return updated, invite
def test_demonstrate_pattern(self):
""" This just demonstrates typical usage """
# setup, pretend this was done before
for i in range(0, 5):
ent = MyModel(val=i)
ent.put()
## pattern starts here ##
# check the ver of particular lock before starting work
verstart = locks.Lock.get("mylock").ver
# now pretend like some other process changed the ver
# after you started working
locks.Lock.incr("mylock")
item = MyModel.query(MyModel.val == 3).get()
def work():
item.val = 100
item.put()
vercheck = locks.Lock.get("mylock").ver
if vercheck != verstart:
# stop right here; don't process those items
# try it again later, or raise an exception, or ...
raise ndb.Rollback("versioning or sequence issue")
ndb.transaction(work, retries=0, xg=True)
## pattern stops here ##
# this just checks that the xaction was rolled back
for item in MyModel.query():
# this is true because that work operation
# never committed
self.assertTrue(item.val < 100)
def post(self):
if not referer_check(self.request):
return
uid = self.session.get('uid')
if not uid:
return
ndb.transaction(lambda: unscribe_transaction(uid))
self.response.write('')
def record(cls, loser, winner, generation):
def _tx():
id = "%d/%d" % (loser.id(), winner.id())
vote = cls.get_by_id(id)
if not vote:
vote = cls(id=id, loser=loser, winner=winner, generation=generation)
vote.count += 1
vote.put()
ndb.transaction(_tx)
def post(self):
if not referer_check(self.request):
return
uid = self.session.get('uid')
endpoint = self.request.get('endpoint')
if not uid or not endpoint:
return
ndb.transaction(lambda: scribe_transaction(uid, endpoint))
self.response.write('')
def replicate_auth_db():
"""Increments auth_db_rev by one.
It is a signal that Auth DB should be replicated to Replicas. If called from
inside a transaction, it inherits it and updates auth_db_rev only once (even
if called multiple times during that transaction).
Should only be called for services in Standalone or Primary modes. Will raise
ValueError if called on Replica. When called for service in Standalone mode,
will update auth_db_rev but won't kick any replication. For services in
Primary mode will also initiate replication by calling callback set in
'configure_as_primary'.
WARNING: This function relies on a valid transaction context. NDB hooks and
asynchronous operations are known to be buggy in this regard: NDB hook for
an async operation in a transaction may be called with a wrong context
(main event loop context instead of transaction context). One way to work
around that is to monkey patch NDB (as done here: https://goo.gl/1yASjL).
Another is to not use hooks at all. There's no way to differentiate between
sync and async modes of an NDB operation from inside a hook. And without a
strict assert it's very easy to forget about "Do not use put_async" warning.
For that reason _post_put_hook is NOT used and replicate_auth_db() should be
called explicitly whenever relevant part of root_key() entity group is
updated.
"""
def increment_revision_and_update_replicas():
"""Does the actual job, called inside a transaction."""
# Update auth_db_rev. replication_state_key() is in same group as root_key.
state = replication_state_key().get()
if not state:
primary_id = app_identity.get_application_id() if is_primary() else None
state = AuthReplicationState(
key=replication_state_key(),
primary_id=primary_id,
auth_db_rev=0)
# Assert Primary or Standalone. Replicas can't increment auth db revision.
if not is_primary() and state.primary_id:
raise ValueError('Can\'t modify Auth DB on Replica')
state.auth_db_rev += 1
state.modified_ts = utils.utcnow()
state.put()
# Only Primary does active replication.
if is_primary():
_replication_callback(state)
# If not in a transaction, start a new one.
if not ndb.in_transaction():
ndb.transaction(increment_revision_and_update_replicas)
return
# If in a transaction, use transaction context to store "already did this"
# flag. Note that each transaction retry gets its own new transaction context,
# see ndb/context.py, 'transaction' tasklet, around line 982 (for SDK 1.9.6).
ctx = ndb.get_context()
if not getattr(ctx, '_auth_db_inc_called', False):
increment_revision_and_update_replicas()
ctx._auth_db_inc_called = True
def _cancel_quotation(order_id):
def trans():
node_order = get_node_order(order_id)
if node_order.odoo_sale_order_id:
update_odoo_quotation(node_order.odoo_sale_order_id, {'state': QuotationState.CANCEL.value})
node_order.populate(status=NodeOrderStatus.CANCELED, cancel_time=now())
node_order.put()
ndb.transaction(trans)
def get(self):
bm = Bookmarks.get_by_id(int(self.request.get('bm')))
if users.get_current_user() == bm.user:
def txn():
bm.url = self.request.get('url').encode('utf8')
bm.title = self.request.get('title').encode('utf8')
bm.comment = self.request.get('comment').encode('utf8')
bm.put()
ndb.transaction(txn)
self.redirect('/')
def api_set_referral(params, user_detail):
def trans():
code = params.get("code")
if not code:
raise ApiCallException(u'Unknown invitation code received')
pp = ProfilePointer.get_by_user_code(code)
if not pp:
raise ApiCallException(u'Unknown invitation code received')
username = get_iyo_username(user_detail)
if username == pp.username:
raise ApiCallException(u'You can\'t use your own invitation code')
my_profile = TffProfile.create_key(username).get() # type: TffProfile
if not my_profile:
raise ApiCallException(u'We were unable to find your profile')
if my_profile.referrer_user:
raise ApiCallException(u'You already set your referrer')
referrer_profile = TffProfile.create_key(pp.username).get()
if not referrer_profile:
raise ApiCallException(
u'We were unable to find your referrer\'s profile')
my_profile.referrer_user = referrer_profile.app_user
my_profile.referrer_username = pp.username
my_profile.put()
deferred.defer(remove_user_from_role,
user_detail,
RogerthatRoles.PUBLIC,
_transactional=True)
deferred.defer(add_user_to_role,
user_detail,
RogerthatRoles.MEMBERS,
_transactional=True)
deferred.defer(remove_user_from_organization,
username,
Organization.PUBLIC,
_transactional=True)
deferred.defer(invite_user_to_organization,
username,
Organization.MEMBERS,
_transactional=True)
deferred.defer(store_referral_in_user_data,
my_profile.key,
_transactional=True)
ndb.transaction(trans, xg=True)
return {
u'result':
u'You successfully joined the ThreeFold community. Welcome aboard!'
}
def post(self):
dom = parseString(self.request.body)
wsRoot = dom.getElementsByTagName('workspace').item(0)
wsName = wsRoot.getAttribute('name')
try:
nextNoteNum = int(wsRoot.getAttribute('nextNoteNum'))
except:
nextNoteNum = 0
notesJsonArray = []
nlNotes = wsRoot.getElementsByTagName('note')
for i in range(nlNotes.length):
node = nlNotes.item(i)
note = {}
for j in range(node.attributes.length):
attr = node.attributes.item(j)
if attr.name in models.Notes.DBKEYS: # only use valid attributes
note[models.Notes.DB2JS[attr.name]] = attr.nodeValue
note['text'] = node.firstChild.nodeValue.strip()
notesJsonArray.append(note)
def txn():
nowtime = datetime.datetime.now().replace(microsecond=0)
workspace = models.Workspace.get_by_wsName(wsName)
if not workspace:
workspace = models.Workspace.create(wsName, nextNoteNum,
nowtime)
workspace.put()
workspace.time = nowtime
workspace.nextNoteNum = nextNoteNum
workspace.put()
notes = models.Notes(workspaceKey=workspace.key,
time=nowtime,
notesJsonArray=notesJsonArray)
notes.put()
for i in range(5):
try:
ndb.transaction(txn, xg=True)
break
except db.TransactionFailedError:
time.sleep(1.1)
origin = self.request.headers.get('Origin', '')
logging.info(origin)
if origin in ('http://localhost:8080', 'http://www.aypwip.org',
'https://www.aypwip.org'):
self.response.headers['Access-Control-Allow-Origin'] = origin
self.response.content_type = 'text/xml'
self.response.write(
'<return><status value="ok" update="%s"/></return>' %
'2017-11-01 12:12:12')
def testInTxn(self):
def AssertInTxn():
self.assertTrue(ndb.in_transaction())
def RunAssert():
fut = utils.GetNoOpFuture()
fut.add_callback(AssertInTxn)
fut.add_immediate_callback(AssertInTxn)
fut.get_result()
ndb.transaction(RunAssert)
def setUp(self):
super(TestOutput, self).setUp()
request = task_request.make_request(_gen_request(), True)
result_summary = task_result.new_result_summary(request)
result_summary.modified_ts = utils.utcnow()
ndb.transaction(result_summary.put)
self.run_result = task_result.new_run_result(request, 1, "localhost", "abc", {})
self.run_result.modified_ts = utils.utcnow()
result_summary.set_from_run_result(self.run_result, request)
ndb.transaction(lambda: ndb.put_multi((result_summary, self.run_result)))
self.run_result = self.run_result.key.get()
def delicious(tag, comment, user):
link = tag.a
bm = Bookmarks()
def txn():
bm.original = link['href']
bm.title = link.text
bm.comment = comment
bm.user = user
bm.put()
ndb.transaction(txn)
deferred.defer(main_parser, bm.key, _target="worker", _queue="parser")
def setUp(self): super(TestOutput, self).setUp() request = task_request.make_request(_gen_request_data()) result_summary = task_result.new_result_summary(request) result_summary.modified_ts = utils.utcnow() ndb.transaction(result_summary.put) self.run_result = task_result.new_run_result(request, 1, 'localhost', 'abc') self.run_result.modified_ts = utils.utcnow() result_summary.set_from_run_result(self.run_result, request) ndb.transaction(lambda: ndb.put_multi((result_summary, self.run_result))) self.run_result = self.run_result.key.get()
def get(self):
old = Bookmarks.get_by_id(int(self.request.get('bm')))
bm = Bookmarks()
def txn():
bm.original = old.original
bm.title = old.title
bm.comment = old.comment
bm.user = users.get_current_user()
bm.put()
ndb.transaction(txn)
deferred.defer(main_parser, bm.key, _queue="parser")
def get(self):
bm = Bookmarks()
def txn():
bm.original = self.request.get('url')
bm.url = self.request.get('url')
bm.title = self.request.get('title')
bm.comment = self.request.get('comment')
bm.user = users.User(str(self.request.get('user')))
bm.put()
ndb.transaction(txn)
main_parser(bm.key)
self.redirect('/')
def flushcache():
# conditionally flush memcache if this is loaded
# due to module deployment
_cachestate = cachestate.get()
if _cachestate.cleartime != gaeutils.App.deploy_time:
logging.debug('flushing cache. last cleared %i and curr deploy time %i' % (
_cachestate.cleartime, gaeutils.App.deploy_time))
def txn():
_cachestate.cleartime = gaeutils.App.deploy_time
_cachestate.put()
if memcache.flush_all():
ndb.transaction(txn)
def post(self):
if not referer_check(self.request):
return
uid = self.session.get('uid')
if not uid:
return
pid = self.request.get('pid')
focuced = False
if self.request.get('v'):
focuced = True
ndb.transaction(lambda: change_focus_transaction(uid, pid, focuced))
self.response.write('')
def sendTestEmail(self, message_entity):
"""Send the test emails to the requested address.
Args:
messages_entity: Messages entity containing the messages to be emailed.
"""
assert access_checker.isSet(self.request_data.program)
test_email_addr = self.cleaned_data.get('test_email', None)
if not test_email_addr:
return
self.request_data.redirect.program()
test_org_key = ndb.Key(
self.request_data.models.org_model._get_kind(),
'/'.join([self.request_data.program.key().name(), TEST_ORG_ID]))
org_app_context = {
'url': links.ABSOLUTE_LINKER.organization(
test_org_key, urls.UrlNames.ORG_PROFILE_EDIT),
'org': TEST_ORG_NAME,
}
proposal_context = {
'proposal_title': TEST_PROPOSAL_TITLE,
'org_entity': TEST_ORG_ENTITY,
}
mail_txns = []
if message_entity.accepted_orgs_msg:
mail_txns.append(self.getSendMailFromTemplateStringTxn(
test_email_addr, notifications.DEF_ACCEPTED_ORG % (org_app_context),
message_entity.accepted_orgs_msg, org_app_context))
if message_entity.rejected_orgs_msg:
mail_txns.append(self.getSendMailFromTemplateStringTxn(
test_email_addr, notifications.DEF_REJECTED_ORG % (org_app_context),
message_entity.rejected_orgs_msg, org_app_context))
if message_entity.accepted_students_msg:
mail_txns.append(self.getSendMailFromTemplateStringTxn(
test_email_addr, 'Congratulations!',
message_entity.accepted_students_msg, proposal_context))
if message_entity.rejected_students_msg:
mail_txns.append(self.getSendMailFromTemplateStringTxn(
test_email_addr,
'Thank you for applying to %s' % (self.request_data.program.name),
message_entity.rejected_students_msg, proposal_context))
for mail_txn in mail_txns:
ndb.transaction(mail_txn)
def reduceProcess(data_id, entities):
# TODO: (Aruna) Fix these import
from melange.logic import cached_list
from melange.utils import lists
ctx = context.get()
params = ctx.mapreduce_spec.mapper.params
list_id = params['list_id']
ndb.transaction(lambda: cached_list.setCacheItems(
data_id, map(json.loads, entities), lists.getList(list_id).valid_period))
def test_new_result_summary(self):
request = task_request.make_request(_gen_request(), True)
actual = task_result.new_result_summary(request)
expected = {
'abandoned_ts': None,
'bot_dimensions': None,
'bot_id': None,
'bot_version': None,
'children_task_ids': [],
'completed_ts': None,
'costs_usd': [],
'cost_saved_usd': None,
'created_ts': self.now,
'deduped_from': None,
'duration': None,
'exit_code': None,
'failure': False,
'id': '1d69b9f088008810',
'internal_failure': False,
'modified_ts': None,
'name': u'Request name',
'outputs_ref': None,
'properties_hash': None,
'server_versions': [],
'started_ts': None,
'state': task_result.State.PENDING,
'tags': [
u'pool:default',
u'priority:50',
u'tag:1',
u'user:Jesus',
],
'try_number': None,
'user': u'Jesus',
}
self.assertEqual(expected, actual.to_dict())
self.assertEqual(50, actual.request.priority)
self.assertEqual(True, actual.can_be_canceled)
actual.state = task_result.State.RUNNING
self.assertEqual(False, actual.can_be_canceled)
actual.children_task_ids = [
'1d69ba3ea8008810',
'3d69ba3ea8008810',
'2d69ba3ea8008810',
]
actual.modified_ts = utils.utcnow()
ndb.transaction(actual.put)
expected = [
u'1d69ba3ea8008810', u'2d69ba3ea8008810', u'3d69ba3ea8008810'
]
self.assertEqual(expected, actual.key.get().children_task_ids)
def _update_transaction_status(transaction_id, status):
def trans():
transaction = PayconiqTransaction.create_key(transaction_id).get()
if transaction.status != PayconiqTransaction.STATUS_PENDING and transaction.status != status:
return
transaction.status = status
transaction.put()
deferred.defer(send_update_payment_status_request_to_user, transaction.app_user, transaction_id, status,
_transactional=True, _queue=FAST_QUEUE)
ndb.transaction(trans)
def delicious(tag, comment, user):
link = tag.a
bm = Bookmarks()
def txn():
bm.original = link['href']
bm.title = link.text
bm.comment = comment
bm.user = user
bm.put()
ndb.transaction(txn)
deferred.defer(main_parser, bm.key, _target="worker", _queue="parser")
def record(cls, loser, winner, generation):
def _tx():
id = "%d/%d" % (loser.id(), winner.id())
vote = cls.get_by_id(id)
if not vote:
vote = cls(id=id,
loser=loser,
winner=winner,
generation=generation)
vote.count += 1
vote.put()
ndb.transaction(_tx)
def reduceProcess(data_id, entities):
# TODO: (Aruna) Fix these import
from melange.logic import cached_list
from melange.utils import lists
ctx = context.get()
params = ctx.mapreduce_spec.mapper.params
list_id = params['list_id']
ndb.transaction(
lambda: cached_list.setCacheItems(data_id, map(json.loads, entities),
lists.getList(list_id).valid_period))
def test_set_from_run_result(self):
request = task_request.make_request(_gen_request(), True)
result_summary = task_result.new_result_summary(request)
run_result = task_result.new_run_result(request, 1, 'localhost', 'abc', {})
self.assertTrue(result_summary.need_update_from_run_result(run_result))
result_summary.modified_ts = utils.utcnow()
run_result.modified_ts = utils.utcnow()
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result)))
self.assertTrue(result_summary.need_update_from_run_result(run_result))
result_summary.set_from_run_result(run_result, request)
ndb.transaction(lambda: ndb.put_multi([result_summary]))
self.assertFalse(result_summary.need_update_from_run_result(run_result))
def test_set_from_run_result(self):
request = mkreq(_gen_request())
result_summary = task_result.new_result_summary(request)
run_result = task_result.new_run_result(request, 1, 'localhost', 'abc', {})
self.assertTrue(result_summary.need_update_from_run_result(run_result))
result_summary.modified_ts = utils.utcnow()
run_result.modified_ts = utils.utcnow()
ndb.transaction(lambda: ndb.put_multi((result_summary, run_result)))
self.assertTrue(result_summary.need_update_from_run_result(run_result))
result_summary.set_from_run_result(run_result, request)
ndb.transaction(lambda: ndb.put_multi([result_summary]))
self.assertFalse(result_summary.need_update_from_run_result(run_result))
def test_run_result_timeout(self): request = task_request.make_request(_gen_request_data()) result_summary = task_result.new_result_summary(request) result_summary.modified_ts = utils.utcnow() ndb.transaction(result_summary.put) run_result = task_result.new_run_result(request, 1, 'localhost', 'abc') run_result.state = task_result.State.TIMED_OUT run_result.completed_ts = utils.utcnow() run_result.modified_ts = utils.utcnow() result_summary.set_from_run_result(run_result, request) ndb.transaction(lambda: ndb.put_multi((run_result, result_summary))) run_result = run_result.key.get() result_summary = result_summary.key.get() self.assertEqual(True, run_result.failure) self.assertEqual(True, result_summary.failure)
def test_run_result_timeout(self):
request = task_request.make_request(_gen_request(), True)
result_summary = task_result.new_result_summary(request)
result_summary.modified_ts = utils.utcnow()
ndb.transaction(result_summary.put)
run_result = task_result.new_run_result(request, 1, 'localhost', 'abc', {})
run_result.state = task_result.State.TIMED_OUT
run_result.completed_ts = utils.utcnow()
run_result.modified_ts = utils.utcnow()
result_summary.set_from_run_result(run_result, request)
ndb.transaction(lambda: ndb.put_multi((run_result, result_summary)))
run_result = run_result.key.get()
result_summary = result_summary.key.get()
self.assertEqual(True, run_result.failure)
self.assertEqual(True, result_summary.failure)
def get(self):
logging.info("REBOOT")
send_to = set()
to_delete = set()
msg = json.dumps({ "reboot": True })
for e in events.query():
for l in e.listeners:
if l not in send_to:
sent_to.add(l)
to_delete.add(e.key)
def delete():
ndb.delete_multi(to_delete)
ndb.transaction(delete)
for l in send_to:
deferred.defer(channel.send_message, str(l), msg)
def test_new_result_summary(self):
request = task_request.make_request(_gen_request(), True)
actual = task_result.new_result_summary(request)
expected = {
'abandoned_ts': None,
'bot_dimensions': None,
'bot_id': None,
'bot_version': None,
'children_task_ids': [],
'completed_ts': None,
'costs_usd': [],
'cost_saved_usd': None,
'created_ts': self.now,
'deduped_from': None,
'duration': None,
'exit_code': None,
'failure': False,
'id': '1d69b9f088008810',
'internal_failure': False,
'modified_ts': None,
'name': u'Request name',
'outputs_ref': None,
'properties_hash': None,
'server_versions': [],
'started_ts': None,
'state': task_result.State.PENDING,
'tags': [
u'pool:default',
u'priority:50',
u'tag:1',
u'user:Jesus',
],
'try_number': None,
'user': u'Jesus',
}
self.assertEqual(expected, actual.to_dict())
self.assertEqual(50, actual.request.priority)
self.assertEqual(True, actual.can_be_canceled)
actual.state = task_result.State.RUNNING
self.assertEqual(False, actual.can_be_canceled)
actual.children_task_ids = [
'1d69ba3ea8008810', '3d69ba3ea8008810', '2d69ba3ea8008810',
]
actual.modified_ts = utils.utcnow()
ndb.transaction(actual.put)
expected = [u'1d69ba3ea8008810', u'2d69ba3ea8008810', u'3d69ba3ea8008810']
self.assertEqual(expected, actual.key.get().children_task_ids)
