def deferred_update(): """ Update all users in the datastore with lowercase nick and full name """ logging.info("Deferred user update starting") CHUNK_SIZE = 200 count = 0 offset = 0 Context.disable_cache() try: q = UserModel.query() while True: ulist = [] chunk = 0 for um in q.fetch(CHUNK_SIZE, offset = offset): chunk += 1 if um.nick_lc is None: try: um.nick_lc = um.nickname.lower() um.name_lc = um.prefs.get("full_name", "").lower() if um.prefs else "" ulist.append(um) except Exception as e: logging.info("Exception in deferred_update() when setting nick_lc: {0}".format(e)) if ulist: try: ndb.put_multi(ulist) count += len(ulist) except Exception as e: logging.info("Exception in deferred_update() when updating ndb: {0}".format(e)) if chunk < CHUNK_SIZE: break offset += CHUNK_SIZE except Exception as e: logging.info("Exception in deferred_update(): {0}, already updated {1} records".format(e, count)) # Do not retry the task raise deferred.PermanentTaskFailure() logging.info("Completed updating {0} user records".format(count))
def deferred_update(): """ Update all users in the datastore with lowercase nick and full name """ logging.info("Deferred user update starting") CHUNK_SIZE = 200 scan = 0 count = 0 with ndb.Client().context(): try: q = UserModel.query() for um in iter_q(q, chunk_size=CHUNK_SIZE): scan += 1 if um.email and not um.email.islower(): um.email = um.email.lower() um.put() count += 1 if scan % 1000 == 0: logging.info("Completed scanning {0} and updating {1} user records".format(scan, count)) except Exception as e: logging.info( "Exception in deferred_update(): {0}, already scanned {1} records and updated {2}" .format(e, scan, count) ) # Do not retry the task # !!! TODO: Alternative solution for Python 3 GAE environment # raise deferred.PermanentTaskFailure() logging.info("Completed scanning {0} and updating {1} user records".format(scan, count))