def logAccess(desti, fro):
try:
db.run_in_transaction(incDestiCounter, desti)
db.run_in_transaction(incFroCounter, fro)
return True
except Exception:
return False
def set_vote(link_id, user, newvote):
"""
Record 'user' casting a 'vote' for a link with an id of 'link_id'.
The 'newvote' is usually an integer in [-1, 0, 1].
"""
if user is None:
return
userid = str(user_account.getUserIDfromObject(user))
def txn():
link = LinkModel.get_by_id(link_id)
vote = VoteModel.get_by_key_name(key_names = userid, parent = link)
if vote is None:
vote = VoteModel(key_name = userid, parent = link)
if vote.vote == newvote:
return
logging.info("link.score = " + str(link.score))
logging.info("vote.vote = " + str(vote.vote))
logging.info("newvote = " + str(newvote))
link.score = link.score + newvote
logging.info("resulting link.score = " + str(link.score))
vote.vote = newvote
db.put([vote, link])
memcache.set("vote|" + userid + "|" + str(link_id), vote.vote)
return link.score
db.run_in_transaction(txn)
_set_progress_hasVoted(userid)
def post(self, data, check, mutator):
"""Get handler for the code sample delete file."""
assert isSet(data.url_project)
try:
id_value = int(data.request.POST['id'])
code_sample = GSoCCodeSample.get_by_id(id_value, data.url_project)
if not code_sample:
raise exception.BadRequest(message='Requested code sample not found')
upload_of_work = code_sample.upload_of_work
def txn():
code_sample.delete()
if upload_of_work:
# this is executed outside of transaction
upload_of_work.delete()
if data.url_project.countCodeSamples() <= 1:
project = GSoCProject.get(data.url_project.key())
project.code_samples_submitted = False
project.put()
db.run_in_transaction(txn)
url = links.LINKER.userId(
data.url_profile.key(), data.url_project.key().id(),
url_names.GSOC_PROJECT_UPDATE)
return http.HttpResponseRedirect(url)
except KeyError:
raise exception.BadRequest(message='id argument missing in POST data')
except ValueError:
raise exception.BadRequest(
message='id argument in POST data is not a number')
def get(self,airlineCode):
####""" data generation hardcoded start"""
y = OntimeStats(airlineCode ='SQ',
departureCity ='here',
arrivalCity ='there',
totalFlights =1)
db.run_in_transaction(y.addFlight)
y.put()
####""" data generation hardcoded end"""
p = OntimeStats.all()
results = p.fetch(100)
for x in results:
self.response.out.write('<html><body><p>FROM: '+x.departureCity+' TO: '+x.arrivalCity+'</p></body></html>')
self.response.out.write('<html><body><p>ON TIME PERCENT: '+str(x.ontime())+'</p></body></html>')
self.response.out.write('<html><body><p>TOTAL FLIGHTS: '+str(x.totalFlights)+'</p></body></html>')
self.response.out.write('<html><body><p>DELAY AVERAGE: '+str(x.delayAverage())+'</p></body></html>')
self.response.out.write('<html><body><p>MAX DELAY: '+str(x.delayMax())+'</p></body></html>')
self.response.out.write('<img src="http://code.google.com/appengine/images/appengine-noborder-120x30.gif"alt="Powered by Google App Engine" />')
for x in results:
db.delete(x)
def toggleFeatured(self, data, value):
"""Makes the project featured.
Args:
data: A RequestData describing the current request.
value: can be either "checked" or "unchecked".
"""
if value != 'checked' and value != 'unchecked':
raise exception.BadRequest(message="Invalid post data.")
if value == 'checked' and not data.url_project.is_featured:
raise exception.BadRequest(message="Invalid post data.")
if value == 'unchecked' and data.url_project.is_featured:
raise exception.BadRequest(message="Invalid post data.")
project_key = data.url_project.key()
def make_featured_txn():
# transactionally get latest version of the project
project = db.get(project_key)
if value == 'unchecked':
project.is_featured = True
elif value == 'checked':
project.is_featured = False
db.put(project)
db.run_in_transaction(make_featured_txn)
def run(self, job_name, sequence_num, time_started, namespace, output):
results = []
# TODO(mgainer): Notice errors earlier in pipeline, and mark job
# as failed in that case as well.
try:
iterator = input_readers.RecordsReader(output, 0)
for item in iterator:
# Map/reduce puts reducer output into blobstore files as a
# string obtained via "str(result)". Use AST as a safe
# alternative to eval() to get the Python object back.
results.append(ast.literal_eval(item))
time_completed = time.time()
with Namespace(namespace):
db.run_in_transaction(
DurableJobEntity._complete_job, job_name, sequence_num,
MapReduceJob.build_output(self.root_pipeline_id, results),
long(time_completed - time_started))
# Don't know what exceptions are currently, or will be in future,
# thrown from Map/Reduce or Pipeline libraries; these are under
# active development.
#
# pylint: disable=broad-except
except Exception, ex:
time_completed = time.time()
with Namespace(namespace):
db.run_in_transaction(
DurableJobEntity._fail_job, job_name, sequence_num,
MapReduceJob.build_output(self.root_pipeline_id, results,
str(ex)),
long(time_completed - time_started))
def removeArtFromTeam(key, teamKey):
#remove model itself
db.run_in_transaction(transactionRemoveArtKeyFromTeam,teamKey,key)
removeArtFromTeamMemcache(teamKey,key)
def testTransaction(self):
"""Executes multiple operations in one transaction."""
class Author(db.Model):
name = db.StringProperty()
class Book(db.Model):
title = db.StringProperty()
marktwain = Author(name="Mark Twain", key_name="marktwain").put()
def tx():
assert db.get(marktwain).name == "Mark Twain"
b = Book(parent=marktwain, title="The Adventures Of Tom Sawyer")
b.put()
c = Book(parent=marktwain, title="The Hitchhiker's Guide to the Galaxy")
c.put()
c.delete()
db.run_in_transaction(tx)
self.assertEqual(1, Author.all().count())
self.assertEqual(1, Book.all().count())
marktwain = Author.get_by_key_name("marktwain")
def query_tx():
query = db.Query()
query.filter("__key__ = ", marktwain.key())
author = query.get()
self.assertRaises(datastore_errors.BadRequestError, db.run_in_transaction, query_tx)
def testLargerTransaction(self):
"""Executes multiple operations in one transaction."""
class Author(db.Model):
name = db.StringProperty()
class Book(db.Model):
title = db.StringProperty()
def tx():
a = Author(name='Mark Twain', key_name='marktwain')
a.put()
b = Book(parent=a, title="The Adventures Of Tom Sawyer")
b.put()
b.delete()
db.run_in_transaction(tx)
self.assertEqual(1, Author.all().count())
self.assertEqual(0, Book.all().count())
marktwain = Author.get_by_key_name('marktwain')
def query_tx():
query = db.Query()
query.filter('__key__ = ', marktwain.key())
author = query.get()
self.assertRaises(
google.appengine.api.datastore_errors.BadRequestError,
db.run_in_transaction, query_tx)
def resetResources(self):
resource_dict = dict()
for r in self.parent().resources:
resource_dict[r] = 0
#TODO: add exception handling
db.run_in_transaction(self.__setResourcesTran, resource_dict)
def run(self):
def tx():
counter = Counter.get_by_key_name('counter')
counter.value += 1
counter.put()
for i in range(100):
db.run_in_transaction(tx)
def toggleStatus(self, value):
"""Toggles the the application state between accept and pending.
Args:
value: can be either "checked" or "unchecked".
"""
assert isSet(self.data.proposal)
if value != "checked" and value != "unchecked":
raise BadRequest("Invalid post data.")
if value == "checked" and not self.data.proposal.accept_as_project:
raise BadRequest("Invalid post data.")
if value == "unchecked" and self.data.proposal.accept_as_project:
raise BadRequest("Invalid post data.")
proposal_key = self.data.proposal.key()
def update_status_txn():
# transactionally get latest version of the proposal
proposal = db.get(proposal_key)
if value == "unchecked":
proposal.accept_as_project = True
elif value == "checked":
proposal.accept_as_project = False
db.put(proposal)
db.run_in_transaction(update_status_txn)
def togglePublicVisibilty(self, value):
"""Toggles the the public visibility of the application.
Args:
value: can be either "checked" or "unchecked".
"""
assert isSet(self.data.proposal)
if value != "checked" and value != "unchecked":
raise BadRequest("Invalid post data.")
if value == "checked" and not self.data.proposal.is_publicly_visible:
raise BadRequest("Invalid post data.")
if value == "unchecked" and self.data.proposal.is_publicly_visible:
raise BadRequest("Invalid post data.")
proposal_key = self.data.proposal.key()
def update_publicly_visibility_txn():
# transactionally get latest version of the proposal
proposal = db.get(proposal_key)
if value == "unchecked":
proposal.is_publicly_visible = True
elif value == "checked":
proposal.is_publicly_visible = False
db.put(proposal)
db.run_in_transaction(update_publicly_visibility_txn)
def toggleIgnoreProposal(self, value):
"""Toggles the ignore status of the proposal.
Args:
value: can be either "checked" or "unchecked".
"""
assert isSet(self.data.proposal)
if value != "checked" and value != "unchecked":
raise BadRequest("Invalid post data.")
if value == "checked" and self.data.proposal.status != "ignored":
raise BadRequest("Invalid post data.")
if value == "unchecked" and self.data.proposal.status not in ["pending", "withdrawn"]:
raise BadRequest("Invalid post data.")
proposal_key = self.data.proposal.key()
def update_status_txn():
# transactionally get latest version of the proposal
proposal = db.get(proposal_key)
if value == "unchecked":
proposal.status = "ignored"
elif value == "checked":
proposal.status = "pending"
db.put(proposal)
db.run_in_transaction(update_status_txn)
def toggleModificationPermission(self, value):
"""Toggles the permission to modify the proposal after proposal deadline.
Args:
value: can be either "checked" or "unchecked".
"""
assert isSet(self.data.proposal)
if value != "checked" and value != "unchecked":
raise BadRequest("Invalid post data.")
if value == "checked" and not self.data.proposal.is_editable_post_deadline:
raise BadRequest("Invalid post data.")
if value == "unchecked" and self.data.proposal.is_editable_post_deadline:
raise BadRequest("Invalid post data.")
proposal_key = self.data.proposal.key()
def update_modification_perm_txn():
# transactionally get latest version of the proposal
proposal = db.get(proposal_key)
if value == "unchecked":
proposal.is_editable_post_deadline = True
elif value == "checked":
proposal.is_editable_post_deadline = False
db.put(proposal)
db.run_in_transaction(update_modification_perm_txn)
def toggleWithdrawProposal(self, value):
"""Toggles the the application state between withdraw and pending.
Args:
value: can be either "checked" or "unchecked".
"""
assert isSet(self.data.proposal)
if value != "checked" and value != "unchecked":
raise BadRequest("Invalid post data.")
if value == "checked" and not self.data.proposal.status == "withdrawn":
raise BadRequest("Invalid post data.")
if value == "unchecked" and self.data.proposal.status == "withdrawn":
raise BadRequest("Invalid post data.")
proposal_key = self.data.proposal.key()
def update_withdraw_status_txn():
# transactionally get latest version of the proposal
proposal = db.get(proposal_key)
if value == "unchecked":
proposal.status = "withdrawn"
elif value == "checked":
proposal.status = "pending"
db.put(proposal)
db.run_in_transaction(update_withdraw_status_txn)
def get_new_session(cls, user, data=None, auto_delete=True,
countdown=settings.TEMPORARY_SESSION_LIFETIME,
additional_tq_url=None, tq_kwargs={}):
from kay.utils import crypto
from google.appengine.api.labs import taskqueue
def txn(id):
key_name = cls.get_key_name(id)
if additional_tq_url is not None:
tq_kwargs.update({'session_key': db.Key.from_path(cls.kind(),
key_name)})
taskqueue.add(url=url_for(additional_tq_url, **tq_kwargs),
transactional=True)
taskqueue.add(url=url_for("_internal/expire_temporary_session",
session_key=db.Key.from_path(cls.kind(),
key_name)),
countdown=countdown, transactional=True)
session = cls.get_by_key_name(key_name)
if session is None:
if data:
session = cls(key_name=key_name, user=user, data=data)
else:
session = cls(key_name=key_name, user=user)
session.put()
return session
else:
raise db.Rollback("The specified id already exists.")
id = crypto.new_iid()
session = db.run_in_transaction(txn, id)
while session is None:
id = crypto.new_iid()
session = db.run_in_transaction(txn, id)
return session
def accept(request, request_id):
if request.method == 'POST':
reguest = RegistrationRequest.get_by_id(request_id)
if reguest:
reguest.status = REQUEST_STATUS['accepted']
reguest.put()
password = passgen()
try:
new_user = create_new_user(
user_name = reguest.email,
password = password,
is_admin = False,
first_name = reguest.name,
telephone = reguest.telephone,
organization = reguest.organization,
address = reguest.address,
email = reguest.email
)
def txn():
taskqueue.add(url=url_for('reguest/admins/send_accept_to_user',
request_id=reguest.key.id(),
password = password),
transactional=True)
db.run_in_transaction(txn)
except DuplicateKeyError:
pass
return redirect(url_for('reguest/admins/index'))
def merged(self, merged):
"""Updates the branch to include pending PatchSets.
This method runs in an independent transaction.
"""
def trans(key):
b = db.get(key)
for ps in merged:
ps_key = ps.key()
if ps_key in b.to_merge:
b.to_merge.remove(ps_key)
if ps_key in b.merging:
b.merging.remove(ps_key)
if ps_key in b.waiting:
b.waiting.remove(ps_key)
if b.status in ('MERGING', 'BUILDING'):
if b.to_merge:
b.status = 'NEEDS_MERGE'
else:
b.status = None
b.put()
db.run_in_transaction(trans, self.key())
def testTransactionalTasks(self):
"""Tests tasks within transactions."""
def my_transaction():
taskqueue.add(url="/path/to/my/worker", transactional=True)
db.run_in_transaction(my_transaction)
def finish_merge(self, success, fail, defer):
"""Update our patchset lists with the results of a merge.
This method runs in an independent transaction.
"""
def trans(key):
b = db.get(key)
rm = []
rm.extend(success)
rm.extend(fail)
for ps in rm:
ps_key = ps.key()
if ps_key in b.to_merge:
b.to_merge.remove(ps_key)
if ps_key in b.merging:
b.merging.remove(ps_key)
if ps_key in b.waiting:
b.waiting.remove(ps_key)
for ps in defer:
ps_key = ps.key()
if ps_key in b.to_merge:
b.to_merge.remove(ps_key)
if ps_key in b.merging:
b.merging.remove(ps_key)
if ps_key not in b.waiting:
b.waiting.append(ps_key)
b.put()
db.run_in_transaction(trans, self.key())
def post(self):
# To prevent CSRF attacks, all requests must be from the task queue
if "X-AppEngine-QueueName" not in self.request.headers:
logging.error("Potential CSRF attack detected")
self.response.set_status(403, message="Potential CSRF attack detected due to missing header.")
return
if not _is_in_progress(self.request.path):
logging.info("Cancelled.")
return
cursor = self.request.get("cursor")
count = self.request.get("count")
if not count:
count = 0
count = int(count)
kind = self.request.get("kind")
query = self.get_keys_query(kind)
if cursor:
query.with_cursor(cursor)
done = False
new_cursor = cursor
# dev server doesn't throw DeadlineExceededError so we do it ourselves
deadline = datetime.datetime.now() + datetime.timedelta(seconds=25)
try:
for key in query:
def do_update():
e = db.get(key)
if self.update(e):
e.put()
if datetime.datetime.now() > deadline:
raise DeadlineExceededError
if self.use_transaction():
db.run_in_transaction(do_update)
else:
do_update()
new_cursor = query.cursor()
count = count + 1
_set_in_progress(self.request.path, False)
logging.info("Finished! %d %s processed.", count, kind)
done = True
except DeadlineExceededError:
pass
except:
logging.exception("Unexpected exception")
finally:
if done:
return
if new_cursor == cursor:
logging.error("Stopped due to lack of progress at %d %s with cursor = %s", count, kind, new_cursor)
_set_in_progress(self.request.path, False)
else:
logging.info("Processed %d %s so far. Continuing in a new task...", count, kind)
new_params = {}
for name, value in self.request.params.items():
new_params[name] = value
new_params["cursor"] = new_cursor
new_params["count"] = count
taskqueue.add(url=self.request.path, params=new_params)
def _unarchive_friend_connection(fsic_archive_key):
app_user = users.User(fsic_archive_key.parent().name())
service_identity_user = users.User(fsic_archive_key.name())
user_data_key = UserDataArchive.createKey(app_user, service_identity_user)
fsic_archive, user_data_archive = db.get([fsic_archive_key, user_data_key])
to_delete = [fsic_archive]
if user_data_archive:
user_data_data = user_data_archive.data
to_delete.append(user_data_archive)
else:
user_data_data = None
makeFriends(service_identity_user, app_user, app_user, None, None, notify_invitee=False, notify_invitor=False,
user_data=user_data_data)
# set disabled and enabled broadcast types
def trans():
fsic = get_friend_serviceidentity_connection(app_user, service_identity_user)
fsic.disabled_broadcast_types = fsic_archive.disabled_broadcast_types
fsic.enabled_broadcast_types = fsic_archive.enabled_broadcast_types
fsic.put()
db.delete(to_delete)
db.run_in_transaction(trans)
def blipChanges(blip_id, key=None, session=None):
'''
Marks a single blip unread for this user. Supply one of the optional
arguments. Supplying session is always more efficient if you
already have the value
@param key=None: a dict containing wave_id, wavelet_id and email
@param session=None: the parent session object
@param blip_id: the blip that is to be marked read
'''
if blip_id == None:
return
if session == None:
session = sessionTools.get(key['wave_id'], key['wavelet_id'], key['email'])
def worker(session, blip_id):
'''
@transaction_safe
'''
settings = get(session)
if settings:
if blip_id in settings.read_blips:
settings.read_blips.remove(blip_id)
put(settings, session.wave_id, session.wavelet_id, session.email)
db.run_in_transaction(worker, session, blip_id)
def quit(user, tag):
"""Transactionally removes a tag for a user."""
def work():
member = Member.get(user)
member.remove_tag(tag)
member.put()
db.run_in_transaction(work)
def set_group_name(self, group_key, name):
# Yeah, this is a race condition - but the worst that can happen is
# you end up with two groups of the same name and have to rename one.
if 0 < MPGroup.all().filter("name =", name).count(1):
raise RestError("There is already a group of that name")
db.run_in_transaction(self._set_group_name, group_key, name)
return {}
def enable(request, key):
if request.method != 'POST':
return MethodNotAllowed()
if request.user.is_anonymous() or not request.user.is_admin:
return Unauthorized()
recipe = models.Recipe.get(key)
if not recipe:
return NotFound()
# Zmień status na włączony
recipe.disabled = False
recipe.put()
if recipe.rec_vis == VISIBILITY[2]:
def utx():
user = PMMUser.get(recipe.parent_key())
user.rec_pub += 1
user.put()
cache_delete('gfcu', recipe.parent_key().name())
db.run_in_transaction(utx)
# na później odłóż indeksację, uaktualnienie tagów i licznika kategorii
deferred_lib.defer(deferred.recipe_update, recipe.key(), 'enable')
request.notifications.success('Przepis włączony!')
return render_json_response({'redirectUrl': url_for('przepisy/disabled_list')})
def increment(name):
def txn():
counter = Cnt.get_by_key_name(name)
if counter is None: counter = Cnt(key_name=name);
counter.count += 1
counter.put()
db.run_in_transaction(txn)
def add_request(properties, context, user, groupname=None):
blip = context.GetBlipById(properties['blipId'])
group = Group.get_by_key_name(Group.get_key_name(groupname))
text = "You(%s) requested adding you to the group: %s.\n" % (user, groupname)
if group:
if user in group.members:
text += "You are already a member of the group: %s." % groupname
elif user in group.applications:
text += ("You have already applied to join the group: %s. "
"Please wait for a moment." % groupname)
elif user in group.banned_addresses:
text += "You are banned from the group: %s." % groupname
else:
from google.appengine.api import mail
from google.appengine.ext import db
body = "%s has requested joining the group: %s.\n" % (user, groupname)
body += "You can moderate this request on following URL:\n"
body += "http://%s.appspot.com/edit_group?key_name=%s" \
% (get_appid(), group.key().name().replace(":", "%3A"))
mail.send_mail(sender=settings.ADMINS[0][1],
to=group.owner.email,
subject="Join request from %s has come" % user,
body=body)
def txn():
g = Group.get(group.key())
g.applications.append(user)
g.put()
db.run_in_transaction(txn)
text += "Request to join this group has been sent!"
else:
text += "However, there is no such group! Sorry."
blip.CreateChild().GetDocument().SetText(text)
def post(self):
author = self.request.get("author")
blob_key = self.request.get("blob_key")
lat = self.request.get("lat")
long = self.request.get("long")
time = self.request.get("time")
rec = int(self.request.get("rec"))
alt = float(self.request.get("alt"))
t_alt = float(self.request.get("t_alt"))
dist = float(self.request.get("dist"))
t_dist = float(self.request.get("t_dist"))
heart = int(self.request.get("heart"))
def txn():
parsedpts = ParsedPts()
parsedpts.author = users.User(author)
parsedpts.blob_key = blob_key
parsedpts.rec = rec
parsedpts.alt = alt
parsedpts.t_alt = t_alt
parsedpts.dist = dist
parsedpts.t_dist = t_dist
parsedpts.heart = heart
parsedpts.date = strptime(time, dateformat)
parsedpts.pt = db.GeoPt(lat, long)
parsedpts.put()
db.run_in_transaction(txn)
if not uservenue and 'location' in j_venue and 'lat' in j_venue['location'] and 'lng' in j_venue['location']:
userinfo.venue_count = userinfo.venue_count + 1
uservenue = UserVenue(parent=userinfo, location = db.GeoPt(j_venue['location']['lat'], j_venue['location']['lng']))
uservenue.venue_guid = str(j_venue['id'])
uservenue.update_location()
uservenue.user = userinfo.user
uservenue.checkin_guid_list = []
if uservenue: # if there's no uservenue by this point, then the venue was missing a location
uservenue.checkin_guid_list.append(str(checkin['id']))
userinfo.checkin_count += 1
def put_updated_uservenue_and_userinfo(uservenue_param, userinfo_param, num_added):
uservenue_param.put()
userinfo_param.put()
return num_added + 1
try:
num_added = db.run_in_transaction(put_updated_uservenue_and_userinfo, uservenue, userinfo, num_added)
except BadRequestError, err:
logging.warning("Database transaction error due to entity restrictions: %s" % err)
else:
logging.error("Venue missing location with JSON: %s" % str(j_venue))
except KeyError:
logging.error("There was a KeyError when processing the response: " + str(history))
raise
return num_added, int(history['checkins']['count'])
def fetch_and_store_checkins_next(userinfo, limit=100):
num_added, num_received = fetch_and_store_checkins(userinfo, limit)
logging.info("num_added = %d, num_received = %d" % (num_added, num_received))
if num_added == 0:
def put_ready_userinfo(userinfo_param):
userinfo_param.is_ready = True
def StartMap(operation_key,
job_name,
handler_spec,
reader_spec,
writer_spec,
mapper_params,
mapreduce_params=None,
queue_name=None,
shard_count=MAPREDUCE_DEFAULT_SHARDS):
"""Start map as part of datastore admin operation.
Will increase number of active jobs inside the operation and start new map.
Args:
operation_key: Key of the DatastoreAdminOperation for current operation.
job_name: Map job name.
handler_spec: Map handler specification.
reader_spec: Input reader specification.
writer_spec: Output writer specification.
mapper_params: Custom mapper parameters.
mapreduce_params: Custom mapreduce parameters.
queue_name: the name of the queue that will be used by the M/R.
shard_count: the number of shards the M/R will try to use.
Returns:
resulting map job id as string.
"""
if not mapreduce_params:
mapreduce_params = {}
mapreduce_params[DatastoreAdminOperation.
PARAM_DATASTORE_ADMIN_OPERATION] = (str(operation_key))
mapreduce_params['done_callback'] = '%s/%s' % (config.BASE_PATH,
MapreduceDoneHandler.SUFFIX)
if queue_name is not None:
mapreduce_params['done_callback_queue'] = queue_name
mapreduce_params['force_writes'] = 'True'
def tx(is_xg_transaction):
"""Start MapReduce job and update datastore admin state.
Args:
is_xg_transaction: True if we are running inside a xg-enabled
transaction, else False if we are running inside a non-xg-enabled
transaction (which means the datastore admin state is updated in one
transaction and the MapReduce job in an indepedent transaction).
Returns:
result MapReduce job id as a string.
"""
job_id = control.start_map(job_name,
handler_spec,
reader_spec,
mapper_params,
output_writer_spec=writer_spec,
mapreduce_parameters=mapreduce_params,
base_path=config.MAPREDUCE_PATH,
shard_count=shard_count,
in_xg_transaction=is_xg_transaction,
queue_name=queue_name)
operation = DatastoreAdminOperation.get(operation_key)
operation.status = DatastoreAdminOperation.STATUS_ACTIVE
operation.active_jobs += 1
operation.active_job_ids = list(
set(operation.active_job_ids + [job_id]))
operation.put(config=_CreateDatastoreConfig())
return job_id
datastore_type = datastore_rpc._GetDatastoreType()
if datastore_type != datastore_rpc.BaseConnection.MASTER_SLAVE_DATASTORE:
return db.run_in_transaction_options(
db.create_transaction_options(xg=True), tx, True)
else:
return db.run_in_transaction(tx, False)
def testAcceptProposalInTxn(self):
# the function should safely execute within a single entity group txn
db.run_in_transaction(proposal_logic.acceptProposal, self.proposal)
def post(self):
"""Mapreduce done callback to delete job data if it was successful."""
if 'Mapreduce-Id' in self.request.headers:
mapreduce_id = self.request.headers['Mapreduce-Id']
mapreduce_state = model.MapreduceState.get_by_job_id(mapreduce_id)
mapreduce_params = mapreduce_state.mapreduce_spec.params
db_config = _CreateDatastoreConfig()
if mapreduce_state.result_status == model.MapreduceState.RESULT_SUCCESS:
operation_key = mapreduce_params.get(
DatastoreAdminOperation.PARAM_DATASTORE_ADMIN_OPERATION)
if operation_key is None:
logging.error(
'Done callback for job %s without operation key.',
mapreduce_id)
else:
def tx():
operation = DatastoreAdminOperation.get(operation_key)
if mapreduce_id in operation.active_job_ids:
operation.active_jobs -= 1
operation.completed_jobs += 1
operation.active_job_ids.remove(mapreduce_id)
if not operation.active_jobs:
if operation.status == DatastoreAdminOperation.STATUS_ACTIVE:
operation.status = DatastoreAdminOperation.STATUS_COMPLETED
db.delete(
DatastoreAdminOperationJob.all().ancestor(
operation),
config=db_config)
operation.put(config=db_config)
if 'done_callback_handler' in mapreduce_params:
done_callback_handler = util.for_name(
mapreduce_params['done_callback_handler'])
if done_callback_handler:
done_callback_handler(operation, mapreduce_id,
mapreduce_state)
else:
logging.error(
'done_callbackup_handler %s was not found',
mapreduce_params['done_callback_handler'])
db.run_in_transaction(tx)
if config.CLEANUP_MAPREDUCE_STATE:
keys = []
shard_states = model.ShardState.find_by_mapreduce_state(
mapreduce_state)
for shard_state in shard_states:
keys.append(shard_state.key())
keys.append(mapreduce_state.key())
keys.append(
model.MapreduceControl.get_key_by_job_id(mapreduce_id))
db.delete(keys, config=db_config)
logging.info('State for successful job %s was deleted.',
mapreduce_id)
else:
logging.info(
'Job %s was not successful so no state was deleted.',
mapreduce_id)
else:
logging.error('Done callback called without Mapreduce Id.')
def goi_account_by_primary_key(user, **kw):
hash = GoogleUser.primary_key(user)
return db.run_in_transaction(GoogleUser.goi_account_by_primary_key_,
user, hash, **kw)
def StartMap(operation_key,
job_name,
handler_spec,
reader_spec,
writer_spec,
mapper_params,
mapreduce_params=None,
start_transaction=True,
queue_name=None):
"""Start map as part of datastore admin operation.
Will increase number of active jobs inside the operation and start new map.
Args:
operation_key: Key of the DatastoreAdminOperation for current operation.
job_name: Map job name.
handler_spec: Map handler specification.
reader_spec: Input reader specification.
writer_spec: Output writer specification.
mapper_params: Custom mapper parameters.
mapreduce_params: Custom mapreduce parameters.
start_transaction: Specify if a new transaction should be started.
queue_name: the name of the queue that will be used by the M/R.
Returns:
resulting map job id as string.
"""
if not mapreduce_params:
mapreduce_params = {}
mapreduce_params[DatastoreAdminOperation.
PARAM_DATASTORE_ADMIN_OPERATION] = (str(operation_key))
mapreduce_params['done_callback'] = '%s/%s' % (config.BASE_PATH,
MapreduceDoneHandler.SUFFIX)
mapreduce_params['force_writes'] = 'True'
def tx():
operation = DatastoreAdminOperation.get(operation_key)
job_id = control.start_map(job_name,
handler_spec,
reader_spec,
mapper_params,
output_writer_spec=writer_spec,
mapreduce_parameters=mapreduce_params,
base_path=config.MAPREDUCE_PATH,
shard_count=DEFAULT_SHARD_SIZE,
transactional=True,
queue_name=queue_name,
transactional_parent=operation)
operation.status = DatastoreAdminOperation.STATUS_ACTIVE
operation.active_jobs += 1
operation.active_job_ids = list(
set(operation.active_job_ids + [job_id]))
operation.put(config=_CreateDatastoreConfig())
return job_id
if start_transaction:
return db.run_in_transaction(tx)
else:
return tx()
def _start_map(cls,
name,
mapper_spec,
mapreduce_params,
base_path=None,
queue_name=None,
eta=None,
countdown=None,
hooks_class_name=None,
_app=None,
transactional=False):
queue_name = queue_name or os.environ.get("HTTP_X_APPENGINE_QUEUENAME",
"default")
if queue_name[0] == "_":
# We are currently in some special queue. E.g. __cron.
queue_name = "default"
# Check that handler can be instantiated.
mapper_spec.get_handler()
# Check that reader can be instantiated and is configured correctly
mapper_input_reader_class = mapper_spec.input_reader_class()
mapper_input_reader_class.validate(mapper_spec)
mapper_output_writer_class = mapper_spec.output_writer_class()
if mapper_output_writer_class:
mapper_output_writer_class.validate(mapper_spec)
mapreduce_id = model.MapreduceState.new_mapreduce_id()
mapreduce_spec = model.MapreduceSpec(name, mapreduce_id,
mapper_spec.to_json(),
mapreduce_params,
hooks_class_name)
kickoff_params = {"mapreduce_spec": mapreduce_spec.to_json_str()}
if _app:
kickoff_params["app"] = _app
kickoff_worker_task = util.HugeTask(url=base_path +
"/kickoffjob_callback",
params=kickoff_params,
eta=eta,
countdown=countdown)
hooks = mapreduce_spec.get_hooks()
config = util.create_datastore_write_config(mapreduce_spec)
def start_mapreduce():
parent = None
if not transactional:
# Save state in datastore so that UI can see it.
# We can't save state in foreign transaction, but conventional UI
# doesn't ask for transactional starts anyway.
state = model.MapreduceState.create_new(
mapreduce_spec.mapreduce_id)
state.mapreduce_spec = mapreduce_spec
state.active = True
state.active_shards = mapper_spec.shard_count
if _app:
state.app_id = _app
state.put(config=config)
parent = state
if hooks is not None:
try:
hooks.enqueue_kickoff_task(kickoff_worker_task, queue_name)
except NotImplementedError:
# Use the default task addition implementation.
pass
else:
return
kickoff_worker_task.add(queue_name,
transactional=True,
parent=parent)
if transactional:
start_mapreduce()
else:
db.run_in_transaction(start_mapreduce)
return mapreduce_id
def insert_or_update(cls, key_name, **kw):
return db.run_in_transaction(insert_or_update_, cls, key_name, **kw)
def post(self, action):
args = {'action': action}
if not self.auth:
args['result'] = 'unauthorized'
elif action == 'register':
args['ident'] = self.request.get('ident')
ident = self.request.get('ident')
args['role'] = self.request.get('role')
role = int(self.request.get('role'))
executive = self.request.get('executive') and True or False
if ident and len(ident) == 5:
query = db.GqlQuery("SELECT * FROM User WHERE ident = :1",
ident)
user = query.get()
if user:
args['result'] = 'duplicated'
elif self.auth.role >= role:
# lower role has greater permission
args['result'] = 'unauthorized'
else:
# gift counter
q = db.GqlQuery('SELECT * FROM Counter WHERE name = :1',
'gift')
counter = q.get()
if not counter:
counter = Counter(name='gift')
counter.put()
count = db.run_in_transaction(inc_counter, counter.key())
# create user
passwd, digest = User.create_passwd()
user = User(ident=ident,
passwd=digest,
role=role,
executive=executive)
user.put()
# create gift
gift = Gift(ident=str(count), giver=user)
gift.put()
self.add_log("user '%s' and gift '%s' registered." %
(user.ident, gift.ident))
args['result'] = 'success'
args['passwd'] = passwd
args['gift'] = gift.ident
else:
args['result'] = 'invalid_ident'
elif action == 'delete':
ident = self.request.get('ident')
args['ident'] = ident
if ident and len(ident) == 5:
query = db.GqlQuery("SELECT * FROM User WHERE ident = :1",
ident)
user = query.get()
if not user:
args['result'] = 'nonexist'
elif self.auth.role >= user.role:
# lower role has greater permission
args['result'] = 'unauthorized'
else:
gift = user.give_set.get()
gift.delete()
user.delete()
self.add_log("user '%s' and gift '%s' deleted." %
(user.ident, gift.ident))
args['result'] = 'success'
else:
args['result'] = 'invalid_ident'
elif action == 'reset':
ident = self.request.get('ident')
args['ident'] = ident
if ident and len(ident) == 5:
query = db.GqlQuery("SELECT * FROM User WHERE ident = :1",
ident)
user = query.get()
if not user:
args['result'] = 'nonexist'
elif self.auth.role >= user.role:
# lower role has greater permission
args['result'] = 'unauthorized'
else:
passwd = user.reset_passwd()
self.add_log("user '%s' password reset." % (user.ident))
args['passwd'] = passwd
args['result'] = 'success'
else:
args['result'] = 'invalid_ident'
elif action == 'update':
ident = self.request.get('ident')
args['ident'] = ident
role = int(self.request.get('role'))
if ident and len(ident) == 5:
query = db.GqlQuery("SELECT * FROM User WHERE ident = :1",
ident)
user = query.get()
if not user:
args['result'] = 'nonexist'
elif self.auth.role >= user.role:
# lower role has greater permission
args['result'] = 'unauthorized'
else:
old_role = user.role
user.role = role
user.put()
self.add_log("user '%s' role changed from %d to %d." %
(user.ident, old_role, user.role))
args['result'] = 'success'
else:
args['result'] = 'invalid_ident'
else:
args['result'] = 'unknown_action'
self.response.headers['Content-Type'] = 'application/json'
self.response.write(json.dumps(args))
def monotonic(self):
id = self.key().id()
return '0|' + str(datetime.today()) + '|' + str(id) + '|' + str(
db.run_in_transaction(Account.monotonic_, id))