Example #1
0
def logAccess(desti, fro):
    try:
        db.run_in_transaction(incDestiCounter, desti)
        db.run_in_transaction(incFroCounter, fro)
        return True
    except Exception:
        return False
def set_vote(link_id, user, newvote):
	"""
	Record 'user' casting a 'vote' for a link with an id of 'link_id'.
	The 'newvote' is usually an integer in [-1, 0, 1].
	"""
	if user is None:
		return
	userid = str(user_account.getUserIDfromObject(user))
  
	def txn():
		link = LinkModel.get_by_id(link_id)
		vote = VoteModel.get_by_key_name(key_names = userid, parent = link)
		if vote is None:
			vote = VoteModel(key_name = userid, parent = link)
		if vote.vote == newvote:
			return
		logging.info("link.score = " + str(link.score))
		logging.info("vote.vote = " + str(vote.vote))
		logging.info("newvote = " + str(newvote))
		link.score = link.score + newvote
		logging.info("resulting link.score = " + str(link.score))
		vote.vote = newvote
		db.put([vote, link])
		memcache.set("vote|" + userid + "|" + str(link_id), vote.vote)
		return link.score

	db.run_in_transaction(txn)
	_set_progress_hasVoted(userid)
Example #3
0
  def post(self, data, check, mutator):
    """Get handler for the code sample delete file."""
    assert isSet(data.url_project)

    try:
      id_value = int(data.request.POST['id'])
      code_sample = GSoCCodeSample.get_by_id(id_value, data.url_project)

      if not code_sample:
        raise exception.BadRequest(message='Requested code sample not found')

      upload_of_work = code_sample.upload_of_work

      def txn():
        code_sample.delete()
        if upload_of_work:
          # this is executed outside of transaction
          upload_of_work.delete()

        if data.url_project.countCodeSamples() <= 1:
          project = GSoCProject.get(data.url_project.key())
          project.code_samples_submitted = False
          project.put()

      db.run_in_transaction(txn)

      url = links.LINKER.userId(
          data.url_profile.key(), data.url_project.key().id(),
          url_names.GSOC_PROJECT_UPDATE)
      return http.HttpResponseRedirect(url)
    except KeyError:
      raise exception.BadRequest(message='id argument missing in POST data')
    except ValueError:
      raise exception.BadRequest(
          message='id argument in POST data is not a number')
Example #4
0
    def get(self,airlineCode):

        ####""" data generation hardcoded start"""
        y = OntimeStats(airlineCode ='SQ',
                        departureCity ='here',
                        arrivalCity ='there',
                        totalFlights =1)
        db.run_in_transaction(y.addFlight)
        y.put()
        ####""" data generation hardcoded end"""

        p = OntimeStats.all()

        results = p.fetch(100)
        for x in results:
            self.response.out.write('<html><body><p>FROM: '+x.departureCity+' TO: '+x.arrivalCity+'</p></body></html>')
            self.response.out.write('<html><body><p>ON TIME PERCENT: '+str(x.ontime())+'</p></body></html>')
            self.response.out.write('<html><body><p>TOTAL FLIGHTS: '+str(x.totalFlights)+'</p></body></html>')
            self.response.out.write('<html><body><p>DELAY AVERAGE: '+str(x.delayAverage())+'</p></body></html>')
            self.response.out.write('<html><body><p>MAX DELAY: '+str(x.delayMax())+'</p></body></html>')
            
        self.response.out.write('<img src="http://code.google.com/appengine/images/appengine-noborder-120x30.gif"alt="Powered by Google App Engine" />')

        for x in results:
            db.delete(x)
Example #5
0
  def toggleFeatured(self, data, value):
    """Makes the project featured.

    Args:
      data: A RequestData describing the current request.
      value: can be either "checked" or "unchecked".
    """
    if value != 'checked' and value != 'unchecked':
      raise exception.BadRequest(message="Invalid post data.")
    if value == 'checked' and not data.url_project.is_featured:
      raise exception.BadRequest(message="Invalid post data.")
    if value == 'unchecked' and data.url_project.is_featured:
      raise exception.BadRequest(message="Invalid post data.")

    project_key = data.url_project.key()

    def make_featured_txn():
      # transactionally get latest version of the project
      project = db.get(project_key)
      if value == 'unchecked':
        project.is_featured = True
      elif value == 'checked':
        project.is_featured = False

      db.put(project)

    db.run_in_transaction(make_featured_txn)
Example #6
0
    def run(self, job_name, sequence_num, time_started, namespace, output):
        results = []

        # TODO(mgainer): Notice errors earlier in pipeline, and mark job
        # as failed in that case as well.
        try:
            iterator = input_readers.RecordsReader(output, 0)
            for item in iterator:
                # Map/reduce puts reducer output into blobstore files as a
                # string obtained via "str(result)".  Use AST as a safe
                # alternative to eval() to get the Python object back.
                results.append(ast.literal_eval(item))
            time_completed = time.time()
            with Namespace(namespace):
                db.run_in_transaction(
                    DurableJobEntity._complete_job, job_name, sequence_num,
                    MapReduceJob.build_output(self.root_pipeline_id, results),
                    long(time_completed - time_started))
        # Don't know what exceptions are currently, or will be in future,
        # thrown from Map/Reduce or Pipeline libraries; these are under
        # active development.
        #
        # pylint: disable=broad-except
        except Exception, ex:
            time_completed = time.time()
            with Namespace(namespace):
                db.run_in_transaction(
                    DurableJobEntity._fail_job, job_name, sequence_num,
                    MapReduceJob.build_output(self.root_pipeline_id, results,
                                              str(ex)),
                    long(time_completed - time_started))
Example #7
0
def removeArtFromTeam(key, teamKey):
    
    
    #remove model itself
    
    db.run_in_transaction(transactionRemoveArtKeyFromTeam,teamKey,key)
    removeArtFromTeamMemcache(teamKey,key)
Example #8
0
    def testTransaction(self):
        """Executes multiple operations in one transaction."""

        class Author(db.Model):
            name = db.StringProperty()

        class Book(db.Model):
            title = db.StringProperty()

        marktwain = Author(name="Mark Twain", key_name="marktwain").put()

        def tx():
            assert db.get(marktwain).name == "Mark Twain"

            b = Book(parent=marktwain, title="The Adventures Of Tom Sawyer")
            b.put()

            c = Book(parent=marktwain, title="The Hitchhiker's Guide to the Galaxy")
            c.put()

            c.delete()

        db.run_in_transaction(tx)

        self.assertEqual(1, Author.all().count())
        self.assertEqual(1, Book.all().count())

        marktwain = Author.get_by_key_name("marktwain")

        def query_tx():
            query = db.Query()
            query.filter("__key__ = ", marktwain.key())
            author = query.get()

        self.assertRaises(datastore_errors.BadRequestError, db.run_in_transaction, query_tx)
    def testLargerTransaction(self):
        """Executes multiple operations in one transaction."""

        class Author(db.Model):
            name = db.StringProperty()

        class Book(db.Model):
            title = db.StringProperty()

        def tx():
            a = Author(name='Mark Twain', key_name='marktwain')
            a.put()

            b = Book(parent=a, title="The Adventures Of Tom Sawyer")
            b.put()

            b.delete()

        db.run_in_transaction(tx)

        self.assertEqual(1, Author.all().count())
        self.assertEqual(0, Book.all().count())

        marktwain = Author.get_by_key_name('marktwain')

        def query_tx():
            query = db.Query()
            query.filter('__key__ = ', marktwain.key())
            author = query.get()

        self.assertRaises(
            google.appengine.api.datastore_errors.BadRequestError,
            db.run_in_transaction, query_tx)
Example #10
0
 def resetResources(self):
     resource_dict = dict()
     for r in self.parent().resources:
         resource_dict[r] = 0
     
     #TODO: add exception handling
     db.run_in_transaction(self.__setResourcesTran, resource_dict)
 def run(self):
     def tx():
         counter = Counter.get_by_key_name('counter')
         counter.value += 1
         counter.put()
     for i in range(100):
         db.run_in_transaction(tx)
Example #12
0
    def toggleStatus(self, value):
        """Toggles the the application state between accept and pending.

    Args:
      value: can be either "checked" or "unchecked".
    """
        assert isSet(self.data.proposal)

        if value != "checked" and value != "unchecked":
            raise BadRequest("Invalid post data.")

        if value == "checked" and not self.data.proposal.accept_as_project:
            raise BadRequest("Invalid post data.")
        if value == "unchecked" and self.data.proposal.accept_as_project:
            raise BadRequest("Invalid post data.")

        proposal_key = self.data.proposal.key()

        def update_status_txn():
            # transactionally get latest version of the proposal
            proposal = db.get(proposal_key)
            if value == "unchecked":
                proposal.accept_as_project = True
            elif value == "checked":
                proposal.accept_as_project = False

            db.put(proposal)

        db.run_in_transaction(update_status_txn)
Example #13
0
    def togglePublicVisibilty(self, value):
        """Toggles the the public visibility of the application.

    Args:
      value: can be either "checked" or "unchecked".
    """
        assert isSet(self.data.proposal)

        if value != "checked" and value != "unchecked":
            raise BadRequest("Invalid post data.")

        if value == "checked" and not self.data.proposal.is_publicly_visible:
            raise BadRequest("Invalid post data.")
        if value == "unchecked" and self.data.proposal.is_publicly_visible:
            raise BadRequest("Invalid post data.")

        proposal_key = self.data.proposal.key()

        def update_publicly_visibility_txn():
            # transactionally get latest version of the proposal
            proposal = db.get(proposal_key)
            if value == "unchecked":
                proposal.is_publicly_visible = True
            elif value == "checked":
                proposal.is_publicly_visible = False

            db.put(proposal)

        db.run_in_transaction(update_publicly_visibility_txn)
Example #14
0
    def toggleIgnoreProposal(self, value):
        """Toggles the ignore status of the proposal.

    Args:
      value: can be either "checked" or "unchecked".
    """
        assert isSet(self.data.proposal)

        if value != "checked" and value != "unchecked":
            raise BadRequest("Invalid post data.")

        if value == "checked" and self.data.proposal.status != "ignored":
            raise BadRequest("Invalid post data.")
        if value == "unchecked" and self.data.proposal.status not in ["pending", "withdrawn"]:
            raise BadRequest("Invalid post data.")

        proposal_key = self.data.proposal.key()

        def update_status_txn():
            # transactionally get latest version of the proposal
            proposal = db.get(proposal_key)
            if value == "unchecked":
                proposal.status = "ignored"
            elif value == "checked":
                proposal.status = "pending"

            db.put(proposal)

        db.run_in_transaction(update_status_txn)
Example #15
0
    def toggleModificationPermission(self, value):
        """Toggles the permission to modify the proposal after proposal deadline.

    Args:
      value: can be either "checked" or "unchecked".
    """
        assert isSet(self.data.proposal)

        if value != "checked" and value != "unchecked":
            raise BadRequest("Invalid post data.")

        if value == "checked" and not self.data.proposal.is_editable_post_deadline:
            raise BadRequest("Invalid post data.")
        if value == "unchecked" and self.data.proposal.is_editable_post_deadline:
            raise BadRequest("Invalid post data.")

        proposal_key = self.data.proposal.key()

        def update_modification_perm_txn():
            # transactionally get latest version of the proposal
            proposal = db.get(proposal_key)
            if value == "unchecked":
                proposal.is_editable_post_deadline = True
            elif value == "checked":
                proposal.is_editable_post_deadline = False

            db.put(proposal)

        db.run_in_transaction(update_modification_perm_txn)
Example #16
0
    def toggleWithdrawProposal(self, value):
        """Toggles the the application state between withdraw and pending.

    Args:
      value: can be either "checked" or "unchecked".
    """
        assert isSet(self.data.proposal)

        if value != "checked" and value != "unchecked":
            raise BadRequest("Invalid post data.")

        if value == "checked" and not self.data.proposal.status == "withdrawn":
            raise BadRequest("Invalid post data.")
        if value == "unchecked" and self.data.proposal.status == "withdrawn":
            raise BadRequest("Invalid post data.")

        proposal_key = self.data.proposal.key()

        def update_withdraw_status_txn():
            # transactionally get latest version of the proposal
            proposal = db.get(proposal_key)
            if value == "unchecked":
                proposal.status = "withdrawn"
            elif value == "checked":
                proposal.status = "pending"

            db.put(proposal)

        db.run_in_transaction(update_withdraw_status_txn)
Example #17
0
  def get_new_session(cls, user, data=None, auto_delete=True,
                      countdown=settings.TEMPORARY_SESSION_LIFETIME,
                      additional_tq_url=None, tq_kwargs={}):
    from kay.utils import crypto
    from google.appengine.api.labs import taskqueue
    def txn(id):
      key_name = cls.get_key_name(id)
      if additional_tq_url is not None:
        tq_kwargs.update({'session_key': db.Key.from_path(cls.kind(),
                                                          key_name)})
        taskqueue.add(url=url_for(additional_tq_url, **tq_kwargs),
                      transactional=True)
      taskqueue.add(url=url_for("_internal/expire_temporary_session",
                                session_key=db.Key.from_path(cls.kind(),
                                                             key_name)),
                    countdown=countdown, transactional=True)
      session = cls.get_by_key_name(key_name)
      if session is None:
        if data:
          session = cls(key_name=key_name, user=user, data=data)
        else:
          session = cls(key_name=key_name, user=user)
        session.put()
        return session
      else:
        raise db.Rollback("The specified id already exists.")

    id = crypto.new_iid()
    session = db.run_in_transaction(txn, id)
    while session is None:
      id = crypto.new_iid()
      session = db.run_in_transaction(txn, id)
    return session
Example #18
0
def accept(request, request_id):
    if request.method == 'POST':
        reguest = RegistrationRequest.get_by_id(request_id)
        if reguest:
            reguest.status = REQUEST_STATUS['accepted']
            reguest.put()
            password = passgen()
            try:
                new_user = create_new_user(
                    user_name = reguest.email,
                    password = password,
                    is_admin = False,
                    first_name = reguest.name,
                    telephone = reguest.telephone,
                    organization = reguest.organization,
                    address = reguest.address,
                    email = reguest.email
                )
                def txn():
                    taskqueue.add(url=url_for('reguest/admins/send_accept_to_user',
                                        request_id=reguest.key.id(),
                                        password = password),
                                        transactional=True)
                db.run_in_transaction(txn)
            except DuplicateKeyError:
                pass
    return redirect(url_for('reguest/admins/index'))
Example #19
0
  def merged(self, merged):
    """Updates the branch to include pending PatchSets.

    This method runs in an independent transaction.
    """
    def trans(key):
      b = db.get(key)

      for ps in merged:
        ps_key = ps.key()

        if ps_key in b.to_merge:
          b.to_merge.remove(ps_key)
        if ps_key in b.merging:
          b.merging.remove(ps_key)
        if ps_key in b.waiting:
          b.waiting.remove(ps_key)

      if b.status in ('MERGING', 'BUILDING'):
        if b.to_merge:
          b.status = 'NEEDS_MERGE'
        else:
          b.status = None
      b.put()
    db.run_in_transaction(trans, self.key())
Example #20
0
    def testTransactionalTasks(self):
        """Tests tasks within transactions."""

        def my_transaction():
            taskqueue.add(url="/path/to/my/worker", transactional=True)

        db.run_in_transaction(my_transaction)
Example #21
0
  def finish_merge(self, success, fail, defer):
    """Update our patchset lists with the results of a merge.

    This method runs in an independent transaction.
    """
    def trans(key):
      b = db.get(key)

      rm = []
      rm.extend(success)
      rm.extend(fail)
      for ps in rm:
        ps_key = ps.key()

        if ps_key in b.to_merge:
          b.to_merge.remove(ps_key)
        if ps_key in b.merging:
          b.merging.remove(ps_key)
        if ps_key in b.waiting:
          b.waiting.remove(ps_key)

      for ps in defer:
        ps_key = ps.key()

        if ps_key in b.to_merge:
          b.to_merge.remove(ps_key)
        if ps_key in b.merging:
          b.merging.remove(ps_key)
        if ps_key not in b.waiting:
          b.waiting.append(ps_key)

      b.put()
    db.run_in_transaction(trans, self.key())
Example #22
0
    def post(self):
        # To prevent CSRF attacks, all requests must be from the task queue
        if "X-AppEngine-QueueName" not in self.request.headers:
            logging.error("Potential CSRF attack detected")
            self.response.set_status(403, message="Potential CSRF attack detected due to missing header.")
            return

        if not _is_in_progress(self.request.path):
            logging.info("Cancelled.")
            return
        cursor = self.request.get("cursor")
        count = self.request.get("count")
        if not count:
            count = 0
        count = int(count)
        kind = self.request.get("kind")
        query = self.get_keys_query(kind)
        if cursor:
            query.with_cursor(cursor)
        done = False
        new_cursor = cursor
        # dev server doesn't throw DeadlineExceededError so we do it ourselves
        deadline = datetime.datetime.now() + datetime.timedelta(seconds=25)
        try:
            for key in query:

                def do_update():
                    e = db.get(key)
                    if self.update(e):
                        e.put()
                    if datetime.datetime.now() > deadline:
                        raise DeadlineExceededError

                if self.use_transaction():
                    db.run_in_transaction(do_update)
                else:
                    do_update()
                new_cursor = query.cursor()
                count = count + 1
            _set_in_progress(self.request.path, False)
            logging.info("Finished! %d %s processed.", count, kind)
            done = True
        except DeadlineExceededError:
            pass
        except:
            logging.exception("Unexpected exception")
        finally:
            if done:
                return
            if new_cursor == cursor:
                logging.error("Stopped due to lack of progress at %d %s with cursor = %s", count, kind, new_cursor)
                _set_in_progress(self.request.path, False)
            else:
                logging.info("Processed %d %s so far.  Continuing in a new task...", count, kind)
                new_params = {}
                for name, value in self.request.params.items():
                    new_params[name] = value
                new_params["cursor"] = new_cursor
                new_params["count"] = count
                taskqueue.add(url=self.request.path, params=new_params)
def _unarchive_friend_connection(fsic_archive_key):
    app_user = users.User(fsic_archive_key.parent().name())
    service_identity_user = users.User(fsic_archive_key.name())

    user_data_key = UserDataArchive.createKey(app_user, service_identity_user)
    fsic_archive, user_data_archive = db.get([fsic_archive_key, user_data_key])

    to_delete = [fsic_archive]
    if user_data_archive:
        user_data_data = user_data_archive.data
        to_delete.append(user_data_archive)
    else:
        user_data_data = None

    makeFriends(service_identity_user, app_user, app_user, None, None, notify_invitee=False, notify_invitor=False,
                user_data=user_data_data)

    # set disabled and enabled broadcast types
    def trans():
        fsic = get_friend_serviceidentity_connection(app_user, service_identity_user)
        fsic.disabled_broadcast_types = fsic_archive.disabled_broadcast_types
        fsic.enabled_broadcast_types = fsic_archive.enabled_broadcast_types
        fsic.put()
        db.delete(to_delete)

    db.run_in_transaction(trans)
Example #24
0
def blipChanges(blip_id, key=None, session=None):
    '''
    Marks a single blip unread for this user. Supply one of the optional
    arguments. Supplying session is always more efficient if you
    already have the value
    @param key=None: a dict containing wave_id, wavelet_id and email
    @param session=None: the parent session object
    @param blip_id: the blip that is to be marked read
    '''
    if blip_id == None:
        return
    if session == None:
        session = sessionTools.get(key['wave_id'], key['wavelet_id'], key['email'])     

    def worker(session, blip_id):
        '''
        @transaction_safe
        '''
        settings = get(session)
        if settings:
            if blip_id in settings.read_blips:
                settings.read_blips.remove(blip_id)
                put(settings, session.wave_id, session.wavelet_id, session.email)

    db.run_in_transaction(worker, session, blip_id)
Example #25
0
 def quit(user, tag):
     """Transactionally removes a tag for a user."""
     def work():
         member = Member.get(user)
         member.remove_tag(tag)
         member.put()
     db.run_in_transaction(work)
Example #26
0
 def set_group_name(self, group_key, name):
   # Yeah, this is a race condition - but the worst that can happen is
   # you end up with two groups of the same name and have to rename one.
   if 0 < MPGroup.all().filter("name =", name).count(1):
     raise RestError("There is already a group of that name")
   db.run_in_transaction(self._set_group_name, group_key, name)
   return {}
Example #27
0
def enable(request, key):   
  if request.method != 'POST':
    return MethodNotAllowed()

  if request.user.is_anonymous() or not request.user.is_admin:
    return Unauthorized()
    
  recipe = models.Recipe.get(key)
  if not recipe:
    return NotFound()
  
  # Zmień status na włączony
  recipe.disabled = False
  recipe.put()

  if recipe.rec_vis == VISIBILITY[2]:
    def utx():
      user = PMMUser.get(recipe.parent_key())
      user.rec_pub += 1
      user.put()
      cache_delete('gfcu', recipe.parent_key().name())
    db.run_in_transaction(utx)

  # na później odłóż indeksację, uaktualnienie tagów i licznika kategorii
  deferred_lib.defer(deferred.recipe_update, recipe.key(), 'enable')

  request.notifications.success('Przepis włączony!')
  return render_json_response({'redirectUrl': url_for('przepisy/disabled_list')})
Example #28
0
def increment(name):
    def txn():
        counter = Cnt.get_by_key_name(name)
        if counter is None: counter = Cnt(key_name=name);
        counter.count += 1
        counter.put()
    db.run_in_transaction(txn)
Example #29
0
def add_request(properties, context, user, groupname=None):
  blip = context.GetBlipById(properties['blipId'])
  group = Group.get_by_key_name(Group.get_key_name(groupname))
  text = "You(%s) requested adding you to the group: %s.\n" % (user, groupname)
  if group:
    if user in group.members:
      text += "You are already a member of the group: %s." % groupname
    elif user in group.applications:
      text += ("You have already applied to join the group: %s. "
               "Please wait for a moment." % groupname)
    elif user in group.banned_addresses:
      text += "You are banned from the group: %s." % groupname
    else:
      from google.appengine.api import mail
      from google.appengine.ext import db
      body = "%s has requested joining the group: %s.\n" % (user, groupname)
      body += "You can moderate this request on following URL:\n"
      body += "http://%s.appspot.com/edit_group?key_name=%s" \
          % (get_appid(), group.key().name().replace(":", "%3A"))
      mail.send_mail(sender=settings.ADMINS[0][1],
                     to=group.owner.email,
                     subject="Join request from %s has come" % user,
                     body=body)
      def txn():
        g = Group.get(group.key())
        g.applications.append(user)
        g.put()
      db.run_in_transaction(txn)
      text += "Request to join this group has been sent!"
  else:
    text += "However, there is no such group! Sorry."
  blip.CreateChild().GetDocument().SetText(text)
Example #30
0
    def post(self):
        author = self.request.get("author")
        blob_key = self.request.get("blob_key")
        lat = self.request.get("lat")
        long = self.request.get("long")
        time = self.request.get("time")
        rec = int(self.request.get("rec"))
        alt = float(self.request.get("alt"))
        t_alt = float(self.request.get("t_alt"))
        dist = float(self.request.get("dist"))
        t_dist = float(self.request.get("t_dist"))
        heart = int(self.request.get("heart"))

        def txn():
            parsedpts = ParsedPts()
            parsedpts.author = users.User(author)
            parsedpts.blob_key = blob_key
            parsedpts.rec = rec
            parsedpts.alt = alt
            parsedpts.t_alt = t_alt
            parsedpts.dist = dist
            parsedpts.t_dist = t_dist
            parsedpts.heart = heart
            parsedpts.date = strptime(time, dateformat)
            parsedpts.pt = db.GeoPt(lat, long)
            parsedpts.put()

        db.run_in_transaction(txn)
          if not uservenue and 'location' in j_venue and 'lat' in j_venue['location'] and 'lng' in j_venue['location']:
            userinfo.venue_count = userinfo.venue_count + 1
            uservenue = UserVenue(parent=userinfo, location = db.GeoPt(j_venue['location']['lat'], j_venue['location']['lng']))
            uservenue.venue_guid = str(j_venue['id'])
            uservenue.update_location()
            uservenue.user = userinfo.user
            uservenue.checkin_guid_list = []
          if uservenue: # if there's no uservenue by this point, then the venue was missing a location
            uservenue.checkin_guid_list.append(str(checkin['id']))
            userinfo.checkin_count += 1
            def put_updated_uservenue_and_userinfo(uservenue_param, userinfo_param, num_added):
              uservenue_param.put()
              userinfo_param.put()
              return num_added + 1
            try:
              num_added = db.run_in_transaction(put_updated_uservenue_and_userinfo, uservenue, userinfo, num_added)
            except BadRequestError, err:
              logging.warning("Database transaction error due to entity restrictions: %s" % err)
          else:
            logging.error("Venue missing location with JSON: %s" % str(j_venue))
  except KeyError:
    logging.error("There was a KeyError when processing the response: " + str(history))
    raise
  return num_added, int(history['checkins']['count'])	

def fetch_and_store_checkins_next(userinfo, limit=100):
  num_added, num_received = fetch_and_store_checkins(userinfo, limit)
  logging.info("num_added = %d, num_received = %d" % (num_added, num_received))
  if num_added == 0:
    def put_ready_userinfo(userinfo_param):
      userinfo_param.is_ready = True
Example #32
0
def StartMap(operation_key,
             job_name,
             handler_spec,
             reader_spec,
             writer_spec,
             mapper_params,
             mapreduce_params=None,
             queue_name=None,
             shard_count=MAPREDUCE_DEFAULT_SHARDS):
    """Start map as part of datastore admin operation.

  Will increase number of active jobs inside the operation and start new map.

  Args:
    operation_key: Key of the DatastoreAdminOperation for current operation.
    job_name: Map job name.
    handler_spec: Map handler specification.
    reader_spec: Input reader specification.
    writer_spec: Output writer specification.
    mapper_params: Custom mapper parameters.
    mapreduce_params: Custom mapreduce parameters.
    queue_name: the name of the queue that will be used by the M/R.
    shard_count: the number of shards the M/R will try to use.

  Returns:
    resulting map job id as string.
  """

    if not mapreduce_params:
        mapreduce_params = {}
    mapreduce_params[DatastoreAdminOperation.
                     PARAM_DATASTORE_ADMIN_OPERATION] = (str(operation_key))
    mapreduce_params['done_callback'] = '%s/%s' % (config.BASE_PATH,
                                                   MapreduceDoneHandler.SUFFIX)
    if queue_name is not None:
        mapreduce_params['done_callback_queue'] = queue_name
    mapreduce_params['force_writes'] = 'True'

    def tx(is_xg_transaction):
        """Start MapReduce job and update datastore admin state.

    Args:
      is_xg_transaction: True if we are running inside a xg-enabled
        transaction, else False if we are running inside a non-xg-enabled
        transaction (which means the datastore admin state is updated in one
        transaction and the MapReduce job in an indepedent transaction).
    Returns:
      result MapReduce job id as a string.
    """
        job_id = control.start_map(job_name,
                                   handler_spec,
                                   reader_spec,
                                   mapper_params,
                                   output_writer_spec=writer_spec,
                                   mapreduce_parameters=mapreduce_params,
                                   base_path=config.MAPREDUCE_PATH,
                                   shard_count=shard_count,
                                   in_xg_transaction=is_xg_transaction,
                                   queue_name=queue_name)
        operation = DatastoreAdminOperation.get(operation_key)
        operation.status = DatastoreAdminOperation.STATUS_ACTIVE
        operation.active_jobs += 1
        operation.active_job_ids = list(
            set(operation.active_job_ids + [job_id]))
        operation.put(config=_CreateDatastoreConfig())
        return job_id

    datastore_type = datastore_rpc._GetDatastoreType()

    if datastore_type != datastore_rpc.BaseConnection.MASTER_SLAVE_DATASTORE:
        return db.run_in_transaction_options(
            db.create_transaction_options(xg=True), tx, True)
    else:
        return db.run_in_transaction(tx, False)
Example #33
0
 def testAcceptProposalInTxn(self):
     # the function should safely execute within a single entity group txn
     db.run_in_transaction(proposal_logic.acceptProposal, self.proposal)
Example #34
0
    def post(self):
        """Mapreduce done callback to delete job data if it was successful."""
        if 'Mapreduce-Id' in self.request.headers:
            mapreduce_id = self.request.headers['Mapreduce-Id']
            mapreduce_state = model.MapreduceState.get_by_job_id(mapreduce_id)
            mapreduce_params = mapreduce_state.mapreduce_spec.params

            db_config = _CreateDatastoreConfig()
            if mapreduce_state.result_status == model.MapreduceState.RESULT_SUCCESS:
                operation_key = mapreduce_params.get(
                    DatastoreAdminOperation.PARAM_DATASTORE_ADMIN_OPERATION)
                if operation_key is None:
                    logging.error(
                        'Done callback for job %s without operation key.',
                        mapreduce_id)
                else:

                    def tx():
                        operation = DatastoreAdminOperation.get(operation_key)
                        if mapreduce_id in operation.active_job_ids:
                            operation.active_jobs -= 1
                            operation.completed_jobs += 1
                            operation.active_job_ids.remove(mapreduce_id)
                        if not operation.active_jobs:
                            if operation.status == DatastoreAdminOperation.STATUS_ACTIVE:
                                operation.status = DatastoreAdminOperation.STATUS_COMPLETED
                            db.delete(
                                DatastoreAdminOperationJob.all().ancestor(
                                    operation),
                                config=db_config)
                        operation.put(config=db_config)
                        if 'done_callback_handler' in mapreduce_params:
                            done_callback_handler = util.for_name(
                                mapreduce_params['done_callback_handler'])
                            if done_callback_handler:
                                done_callback_handler(operation, mapreduce_id,
                                                      mapreduce_state)
                            else:
                                logging.error(
                                    'done_callbackup_handler %s was not found',
                                    mapreduce_params['done_callback_handler'])

                    db.run_in_transaction(tx)
                if config.CLEANUP_MAPREDUCE_STATE:
                    keys = []
                    shard_states = model.ShardState.find_by_mapreduce_state(
                        mapreduce_state)
                    for shard_state in shard_states:
                        keys.append(shard_state.key())

                    keys.append(mapreduce_state.key())
                    keys.append(
                        model.MapreduceControl.get_key_by_job_id(mapreduce_id))
                    db.delete(keys, config=db_config)
                    logging.info('State for successful job %s was deleted.',
                                 mapreduce_id)
            else:
                logging.info(
                    'Job %s was not successful so no state was deleted.',
                    mapreduce_id)
        else:
            logging.error('Done callback called without Mapreduce Id.')
Example #35
0
 def goi_account_by_primary_key(user, **kw):
     hash = GoogleUser.primary_key(user)
     return db.run_in_transaction(GoogleUser.goi_account_by_primary_key_,
                                  user, hash, **kw)
Example #36
0
def StartMap(operation_key,
             job_name,
             handler_spec,
             reader_spec,
             writer_spec,
             mapper_params,
             mapreduce_params=None,
             start_transaction=True,
             queue_name=None):
    """Start map as part of datastore admin operation.

  Will increase number of active jobs inside the operation and start new map.

  Args:
    operation_key: Key of the DatastoreAdminOperation for current operation.
    job_name: Map job name.
    handler_spec: Map handler specification.
    reader_spec: Input reader specification.
    writer_spec: Output writer specification.
    mapper_params: Custom mapper parameters.
    mapreduce_params: Custom mapreduce parameters.
    start_transaction: Specify if a new transaction should be started.
    queue_name: the name of the queue that will be used by the M/R.

  Returns:
    resulting map job id as string.
  """

    if not mapreduce_params:
        mapreduce_params = {}
    mapreduce_params[DatastoreAdminOperation.
                     PARAM_DATASTORE_ADMIN_OPERATION] = (str(operation_key))
    mapreduce_params['done_callback'] = '%s/%s' % (config.BASE_PATH,
                                                   MapreduceDoneHandler.SUFFIX)
    mapreduce_params['force_writes'] = 'True'

    def tx():
        operation = DatastoreAdminOperation.get(operation_key)

        job_id = control.start_map(job_name,
                                   handler_spec,
                                   reader_spec,
                                   mapper_params,
                                   output_writer_spec=writer_spec,
                                   mapreduce_parameters=mapreduce_params,
                                   base_path=config.MAPREDUCE_PATH,
                                   shard_count=DEFAULT_SHARD_SIZE,
                                   transactional=True,
                                   queue_name=queue_name,
                                   transactional_parent=operation)
        operation.status = DatastoreAdminOperation.STATUS_ACTIVE
        operation.active_jobs += 1
        operation.active_job_ids = list(
            set(operation.active_job_ids + [job_id]))
        operation.put(config=_CreateDatastoreConfig())
        return job_id

    if start_transaction:
        return db.run_in_transaction(tx)
    else:
        return tx()
Example #37
0
    def _start_map(cls,
                   name,
                   mapper_spec,
                   mapreduce_params,
                   base_path=None,
                   queue_name=None,
                   eta=None,
                   countdown=None,
                   hooks_class_name=None,
                   _app=None,
                   transactional=False):
        queue_name = queue_name or os.environ.get("HTTP_X_APPENGINE_QUEUENAME",
                                                  "default")
        if queue_name[0] == "_":
            # We are currently in some special queue. E.g. __cron.
            queue_name = "default"

        # Check that handler can be instantiated.
        mapper_spec.get_handler()

        # Check that reader can be instantiated and is configured correctly
        mapper_input_reader_class = mapper_spec.input_reader_class()
        mapper_input_reader_class.validate(mapper_spec)

        mapper_output_writer_class = mapper_spec.output_writer_class()
        if mapper_output_writer_class:
            mapper_output_writer_class.validate(mapper_spec)

        mapreduce_id = model.MapreduceState.new_mapreduce_id()
        mapreduce_spec = model.MapreduceSpec(name, mapreduce_id,
                                             mapper_spec.to_json(),
                                             mapreduce_params,
                                             hooks_class_name)

        kickoff_params = {"mapreduce_spec": mapreduce_spec.to_json_str()}
        if _app:
            kickoff_params["app"] = _app
        kickoff_worker_task = util.HugeTask(url=base_path +
                                            "/kickoffjob_callback",
                                            params=kickoff_params,
                                            eta=eta,
                                            countdown=countdown)

        hooks = mapreduce_spec.get_hooks()
        config = util.create_datastore_write_config(mapreduce_spec)

        def start_mapreduce():
            parent = None
            if not transactional:
                # Save state in datastore so that UI can see it.
                # We can't save state in foreign transaction, but conventional UI
                # doesn't ask for transactional starts anyway.
                state = model.MapreduceState.create_new(
                    mapreduce_spec.mapreduce_id)
                state.mapreduce_spec = mapreduce_spec
                state.active = True
                state.active_shards = mapper_spec.shard_count
                if _app:
                    state.app_id = _app
                state.put(config=config)
                parent = state

            if hooks is not None:
                try:
                    hooks.enqueue_kickoff_task(kickoff_worker_task, queue_name)
                except NotImplementedError:
                    # Use the default task addition implementation.
                    pass
                else:
                    return
            kickoff_worker_task.add(queue_name,
                                    transactional=True,
                                    parent=parent)

        if transactional:
            start_mapreduce()
        else:
            db.run_in_transaction(start_mapreduce)

        return mapreduce_id
Example #38
0
def insert_or_update(cls, key_name, **kw):
    return db.run_in_transaction(insert_or_update_, cls, key_name, **kw)
Example #39
0
    def post(self, action):
        args = {'action': action}

        if not self.auth:
            args['result'] = 'unauthorized'
        elif action == 'register':
            args['ident'] = self.request.get('ident')
            ident = self.request.get('ident')
            args['role'] = self.request.get('role')
            role = int(self.request.get('role'))
            executive = self.request.get('executive') and True or False

            if ident and len(ident) == 5:
                query = db.GqlQuery("SELECT * FROM User WHERE ident = :1",
                                    ident)
                user = query.get()

                if user:
                    args['result'] = 'duplicated'
                elif self.auth.role >= role:
                    # lower role has greater permission
                    args['result'] = 'unauthorized'
                else:
                    # gift counter
                    q = db.GqlQuery('SELECT * FROM Counter WHERE name = :1',
                                    'gift')
                    counter = q.get()
                    if not counter:
                        counter = Counter(name='gift')
                        counter.put()
                    count = db.run_in_transaction(inc_counter, counter.key())

                    # create user
                    passwd, digest = User.create_passwd()
                    user = User(ident=ident,
                                passwd=digest,
                                role=role,
                                executive=executive)
                    user.put()

                    # create gift
                    gift = Gift(ident=str(count), giver=user)
                    gift.put()

                    self.add_log("user '%s' and gift '%s' registered." %
                                 (user.ident, gift.ident))

                    args['result'] = 'success'
                    args['passwd'] = passwd
                    args['gift'] = gift.ident
            else:
                args['result'] = 'invalid_ident'
        elif action == 'delete':
            ident = self.request.get('ident')
            args['ident'] = ident

            if ident and len(ident) == 5:
                query = db.GqlQuery("SELECT * FROM User WHERE ident = :1",
                                    ident)
                user = query.get()

                if not user:
                    args['result'] = 'nonexist'
                elif self.auth.role >= user.role:
                    # lower role has greater permission
                    args['result'] = 'unauthorized'
                else:
                    gift = user.give_set.get()
                    gift.delete()
                    user.delete()
                    self.add_log("user '%s' and gift '%s' deleted." %
                                 (user.ident, gift.ident))
                    args['result'] = 'success'
            else:
                args['result'] = 'invalid_ident'
        elif action == 'reset':
            ident = self.request.get('ident')
            args['ident'] = ident

            if ident and len(ident) == 5:
                query = db.GqlQuery("SELECT * FROM User WHERE ident = :1",
                                    ident)
                user = query.get()

                if not user:
                    args['result'] = 'nonexist'
                elif self.auth.role >= user.role:
                    # lower role has greater permission
                    args['result'] = 'unauthorized'
                else:
                    passwd = user.reset_passwd()
                    self.add_log("user '%s' password reset." % (user.ident))
                    args['passwd'] = passwd
                    args['result'] = 'success'
            else:
                args['result'] = 'invalid_ident'
        elif action == 'update':
            ident = self.request.get('ident')
            args['ident'] = ident
            role = int(self.request.get('role'))

            if ident and len(ident) == 5:
                query = db.GqlQuery("SELECT * FROM User WHERE ident = :1",
                                    ident)
                user = query.get()

                if not user:
                    args['result'] = 'nonexist'
                elif self.auth.role >= user.role:
                    # lower role has greater permission
                    args['result'] = 'unauthorized'
                else:
                    old_role = user.role
                    user.role = role
                    user.put()

                    self.add_log("user '%s' role changed from %d to %d." %
                                 (user.ident, old_role, user.role))
                    args['result'] = 'success'
            else:
                args['result'] = 'invalid_ident'
        else:
            args['result'] = 'unknown_action'

        self.response.headers['Content-Type'] = 'application/json'
        self.response.write(json.dumps(args))
Example #40
0
 def monotonic(self):
     id = self.key().id()
     return '0|' + str(datetime.today()) + '|' + str(id) + '|' + str(
         db.run_in_transaction(Account.monotonic_, id))