def meeting_date_count_deferred(start_cursor): start_time = time.time() # collect meetings per scoutgroup and day, count number of trooppersons and meeting attendees meeting_date_counter_dict = memcache.get("meeting_date_counter_dict") # dict: datetime => number of attendee that day # {"Scoutgroup X": {meetings:[{"date": 2020-01-01, pers:"23"}, {"date": 2020-01-02, pers:"33"}]}, "Scoutgroup Y": {meetings:[{"date": 2020-01-01, pers:"23"}, {"date": 2020-01-02, pers:"33"}]}} there_is_more = true while there_is_more: sgs, start_cursor, there_is_more = ScoutGroup.query().fetch_page(page_size=1, start_cursor=start_cursor): for sg in sgs: logging.info("Updating:%s" % (semester.getname())) for semester in Semester.query().fetch(): for troop_key in Troop.query(Troop.scoutgroup==sg.key, Troop.semester_key==semester.key).fetch(keys_only=True): #troopPersonKeys = TroopPerson.query(TroopPerson.troop == troop.key).fetch(keys_only=True) #numTroopPersons = len(TroopPerson) for meeting in Meeting.query(Meeting.troop == troop_key).fetch(): attending_count = len(meeting.attendingPersons) meeting_day = meeting.datetime.strftime('%Y-%m-%d') if meeting_date_counter_dict.has_key(meeting_day): attending_count += meeting_date_counter_dict[meeting_day] meeting_date_counter_dict[meeting_day] = attending_count if there_is_more: memcache.delete("meeting_date_counter_dict") memcache.add("meeting_date_counter_dict", meeting_date_counter_dict) deferred.defer(meeting_date_count_deferred, start_cursor) meeting_date_counter = [{"date" : day_key, "delt": meeting_date_counter_dict[day_key]} for day_key in sorted(meeting_date_counter_dict)] logging.info("meeting_date_count update done") memcache.delete("meeting_date_count") memcache.add("meeting_date_count", meeting_date_counter) memcache.delete("update_meeting_date_count_is_running")
def dosettroopsemester(): semester_key = Semester.getOrCreateCurrent().key troops = Troop.query().fetch() for troop in troops: #if troop.semester_key != semester_key: troop.semester_key = semester_key logging.info("updating semester for: %s", troop.getname()) troop.put()
def DeleteAllData(): entries = [] entries.extend(Person.query().fetch(keys_only=True)) entries.extend(Troop.query().fetch(keys_only=True)) entries.extend(ScoutGroup.query().fetch(keys_only=True)) entries.extend(Meeting.query().fetch(keys_only=True)) entries.extend(TroopPerson.query().fetch(keys_only=True)) entries.extend(Semester.query().fetch(keys_only=True)) entries.extend(TaskProgress.query().fetch(keys_only=True)) entries.extend(UserPrefs.query().fetch(keys_only=True)) ndb.delete_multi(entries) ndb.get_context().clear_cache() # clear memcache
def update_person_ids(commit, sgroup_key, start_cursor, stage, taskProgress): start_time = time.time() time_is_out = False there_is_more = True oldToNewDict = {} # --- Stage 0 --- if stage == 0: while there_is_more: if time.time() - start_time > max_time_seconds: time_is_out = True break # time out person_keys, start_cursor, there_is_more = Person.query(Person.scoutgroup ==sgroup_key) \ .fetch_page(page_size=PAGE_SIZE, start_cursor=start_cursor, keys_only=True) for person_key in person_keys: if len(str(person_key.id())) < 12: # cannot be a personnr if shorter than 12 chars taskProgress.info("Not updating id=%s" % (str(person_key.id()))) continue person = person_key.get() if person.version == 1 or person.version == 2: taskProgress.info("Already updated id=%s" % (str(person_key.id()))) continue if person.member_no is None: taskProgress.warning("Cannot update, member_no is None id=%s, %s" % (str(person_key.id()), person.getname())) continue taskProgress.info("Updating Person: id=%s, pnr=%s, no=%s, %s" % (str(person.key.id()), person.personnr, str(person.member_no), person.getname())) newPerson, pendingKeyChange = person.updateKey() if newPerson is not None: if commit and newPerson._dirty: k = newPerson.put() assert(k == newPerson.key) if commit and person._dirty: person.put() if pendingKeyChange is not None: if pendingKeyChange._dirty: pendingKeyChange.put() # safe to put, to be able to test the rest of the method if there_is_more: return start_cursor, there_is_more, stage else: taskProgress.info("Updated all persons Persons") stage = 1 start_cursor = None there_is_more = True return start_cursor, there_is_more, stage # restart to start fresh and let datastore catch up taskProgress.info("Loading PendingPersonKeyChange records") ppkc_more = True ppkc_start_cursor = None while ppkc_more: ppkcs, ppkc_start_cursor, ppkc_more = PendingPersonKeyChange.query().fetch_page(page_size=1000, start_cursor=ppkc_start_cursor) for ppkc in ppkcs: oldToNewDict[ppkc.old_key] = ppkc.new_key taskProgress.info("Loaded persons to convert %d" % (len(oldToNewDict))) taskProgress.info("Number of persons to convert %d" % (len(oldToNewDict))) # --- Stage 1 --- if stage == 1: num_meetings_updated = 0 troop_count = 0 there_is_more = True taskProgress.info("Updating Meetings") while there_is_more: if time.time() - start_time > max_time_seconds: return start_cursor, there_is_more, stage troop_keys, start_cursor, there_is_more = Troop.query(Troop.scoutgroup == sgroup_key) \ .fetch_page(page_size=5, start_cursor=start_cursor, keys_only=True) for troop_key in troop_keys: troop_count += 1 meetings = Meeting.query(Meeting.troop == troop_key).fetch() for meeting in meetings: if meeting.uppdateOldPersonKeys(oldToNewDict): num_meetings_updated += 1 if commit: meeting.put() if num_meetings_updated % 10 == 0: taskProgress.info("Updated %d meetings, %d troop_count" % (num_meetings_updated, troop_count)) if there_is_more: return start_cursor, there_is_more, stage else: taskProgress.info("Updated all meetings, time taken: %s s" % (str(time.time() - start_time))) stage = 2 there_is_more = True start_cursor = None return start_cursor, there_is_more, stage # --- Stage 2 --- if stage == 2: update_counter = 0 if start_cursor == None: start_cursor = 0 taskProgress.info("Updating TroopPersons") for index, (old_person_key, new_person_key) in enumerate(oldToNewDict.items()): if index < start_cursor: continue for troopPerson in TroopPerson.query(TroopPerson.person==old_person_key).fetch(): troopPerson.person = new_person_key update_counter += 1 if commit: troopPerson.put() if update_counter % 10 == 0: taskProgress.info("Updated %d TroopPersons" % (update_counter)) if time.time() - start_time > max_time_seconds: start_cursor = index there_is_more = True return start_cursor, there_is_more, stage taskProgress.info("Updated all TroopPersons") there_is_more = True stage = 3 start_cursor = None return start_cursor, there_is_more, stage # restart to start fresh and let datastore catch up # --- Stage 3 --- if stage == 3: taskProgress.info("Deleting old persons") if commit and len(oldToNewDict) != 0: old_person_keys = [old_person_key for index, (old_person_key, new_person_key) in enumerate(oldToNewDict.items())] ndb.delete_multi(old_person_keys) taskProgress.info("Deleted %d old persons" % (len(old_person_keys))) there_is_more = False start_cursor = None stage = 4 taskProgress.info("Done updating Person keys!") return None, False, stage
def merge_scout_group(oldname, newname, commit, taskProgress, user, move_users, move_persons, move_troops, delete_sg, semester_id): start_time = time.time() oldsg = ScoutGroup.getbyname(oldname) if oldsg is None: raise RuntimeError("Old sg name:%s not found" % oldname) newsg = ScoutGroup.getbyname(newname) if newsg is None: raise RuntimeError("New sg name:%s not found" % newname) if not commit: taskProgress.info("*** testmode ***") keys_to_delete = [] entities_to_put_first = [] entities_to_put = [] taskProgress.info("termin: %s" % semester_id) convertsemester = Semester.getbyId(semester_id) if convertsemester is None: taskProgress.error("termin: %s does not exist" % semester_id) currentsemester = Semester.getOrCreateCurrent() if move_users: taskProgress.info("Update all users to the new scoutgroup") for u in UserPrefs.query(UserPrefs.groupaccess == oldsg.key).fetch(): logging.info(" * * moving user for %s" % (u.getname())) u.groupaccess = newsg.key u.activeSemester = currentsemester.key entities_to_put.append(u) if move_persons: taskProgress.info("Moving all persons to the new ScoutGroup:%s" % newsg.getname()) for oldp in Person.query(Person.scoutgroup == oldsg.key).fetch(): oldp.scoutgroup = newsg.key entities_to_put.append(oldp) if move_troops: taskProgress.info("Move all troops to the new ScoutGroup:%s, semester: %s" % (newsg.getname(), convertsemester.getname())) for oldt in Troop.query(Troop.scoutgroup == oldsg.key, Troop.semester_key == convertsemester.key).fetch(): taskProgress.info(" * found old troop for %s, semester=%s" % (str(oldt.key.id()), oldt.semester_key.get().getname())) keys_to_delete.append(oldt.key) newt = Troop.get_by_id(Troop.getid(oldt.scoutnetID, newsg.key, oldt.semester_key), use_memcache=True) if newt is None: taskProgress.info(" * * creating new troop for %s, semester=%s" % (str(oldt.key.id()), oldt.semester_key.get().getname())) newt = Troop.create(oldt.name, oldt.scoutnetID, newsg.key, oldt.semester_key) entities_to_put_first.append(newt) # put first to be able to reference it else: taskProgress.info(" * * already has new troop for %s, semester=%s" % (str(newt.key.id()), newt.semester_key.get().getname())) taskProgress.info(" * Move all trooppersons to the new group") for oldtp in TroopPerson.query(TroopPerson.troop == oldt.key).fetch(): keys_to_delete.append(oldtp.key) newtp = TroopPerson.get_by_id(TroopPerson.getid(newt.key, oldtp.person), use_memcache=True) if newtp is None: logging.info(" * * creating new TroopPerson for %s:%s" % (newt.getname(), oldtp.getname())) newtp = TroopPerson.create_or_update(newt.key, oldtp.person, oldtp.leader) entities_to_put.append(newtp) else: logging.info(" * * already has TroopPerson for %s:%s" % (newt.getname(), oldtp.getname())) taskProgress.info(" * Move all old meetings to the new troop") for oldm in Meeting.query(Meeting.troop==oldt.key).fetch(): keys_to_delete.append(oldm.key) newm = Meeting.get_by_id(Meeting.getId(oldm.datetime, newt.key), use_memcache=True) if newm is None: logging.info(" * * creating new Meeting for %s:%s" % (newt.getname(), oldm.datetime.strftime("%Y-%m-%d %H:%M"))) newm = Meeting(id=Meeting.getId(oldm.datetime, newt.key), datetime=oldm.datetime, name=oldm.name, troop=newt.key, duration=oldm.duration, ishike=oldm.ishike) newm.attendingPersons = oldm.attendingPersons entities_to_put.append(newm) else: logging.info(" * * merging Meeting %s->%s :%s" % (oldm.getname(), newm.getname(), oldm.datetime.strftime("%Y-%m-%d %H:%M"))) need_to_put = False if len(oldm.name) > len(newm.name): # take the longer name. newm.name = oldm.name need_to_put = True for oldattendingpersonkey in oldm.attendingPersons: if oldattendingpersonkey not in newm.attendingPersons: newm.attendingPersons.append(oldattendingpersonkey) need_to_put = True if need_to_put: entities_to_put.append(newm) if delete_sg: keys_to_delete.append(oldsg.key) taskProgress.info("time before put: %s s" % str(time.time() - start_time)) logging.info("Putting %d entities first" % len(entities_to_put_first)) if commit: ndb.put_multi(entities_to_put_first) logging.info("Putting %d entities" % len(entities_to_put)) if commit: ndb.put_multi(entities_to_put) logging.info("Deleting %d keys" % len(keys_to_delete)) if commit: ndb.delete_multi(keys_to_delete) logging.info("clear memcache") if commit: ndb.get_context().clear_cache() taskProgress.info("Done! time: %s s" % str(time.time() - start_time))
def merge_sg(oldname, newname, commit): oldsg = ScoutGroup.getbyname(oldname) if oldsg is None: raise RuntimeError("Old sg name:%s not found" % oldname) newsg = ScoutGroup.getbyname(newname) if newsg is None: raise RuntimeError("New sg name:%s not found" % newname) if not commit: logging.info("*** testmode ***") keys_to_delete = [] entities_to_put_first = [] entities_to_put = [] semester = Semester.getOrCreateCurrent() keys_to_delete.append(oldsg.key) logging.info("Update all users to the new scoutgroup") for u in UserPrefs.query(UserPrefs.groupaccess == oldsg.key).fetch(): u.groupaccess = newsg.key u.activeSemester = semester.key entities_to_put.append(u) logging.info("Moving all persons to the new ScoutGroup:%s", newsg.getname()) for oldp in Person.query(Person.scoutgroup == oldsg.key).fetch(): logging.info(" * Moving %s %s", oldp.getname(), oldp.personnr) oldp.scoutgroup = newsg.key entities_to_put.append(oldp) logging.info("Move all troops to the new ScoutGroup:%s", newsg.getname()) for oldt in Troop.query(Troop.scoutgroup == oldsg.key).fetch(): logging.info(" * found old troop for %s, semester=%s", str(oldt.key.id()), oldt.semester_key.get().getname()) keys_to_delete.append(oldt.key) newt = Troop.get_by_id(Troop.getid(oldt.scoutnetID, newsg.key, oldt.semester_key), use_memcache=True) if newt is None: logging.info(" * * creating new troop for %s, semester=%s", str(oldt.key.id()), oldt.semester_key.get().getname()) newt = Troop.create(oldt.name, oldt.scoutnetID, newsg.key, oldt.semester_key) entities_to_put_first.append(newt) # put first to be able to reference it else: logging.info(" * * already has new troop for %s, semester=%s", str(newt.key.id()), newt.semester_key.get().getname()) logging.info(" * Move all trooppersons to the new group (it they don't already exist there)") for oldtp in TroopPerson.query(TroopPerson.troop == oldt.key).fetch(): keys_to_delete.append(oldtp.key) newtp = TroopPerson.get_by_id(TroopPerson.getid(newt.key, oldtp.person), use_memcache=True) if newtp is None: logging.info(" * * creating new TroopPerson for %s:%s", newt.getname(), oldtp.getname()) newtp = TroopPerson.create(newt.key, oldtp.person, oldtp.leader) entities_to_put.append(newtp) else: logging.info(" * * already has TroopPerson for %s:%s", newt.getname(), oldtp.getname()) logging.info(" * Move all old meetings to the new troop") for oldm in Meeting.query(Meeting.troop==oldt.key).fetch(): keys_to_delete.append(oldm.key) newm = Meeting.get_by_id(Meeting.getId(oldm.datetime, newt.key), use_memcache=True) if newm is None: logging.info(" * * creating new Meeting for %s:%s", newt.getname(), oldm.datetime.strftime("%Y-%m-%d %H:%M")) newm = Meeting.getOrCreate(newt.key, oldm.name, oldm.datetime, oldm.duration, oldm.ishike) newm.attendingPersons = oldm.attendingPersons entities_to_put.append(newm) else: logging.info(" * * already has Meeting for %s:%s", newt.getname(), newt.datetime.strftime("%Y-%m-%d %H:%M")) logging.info("Putting %d entities first", len(entities_to_put_first)) if commit: ndb.put_multi(entities_to_put_first) logging.info("Putting %d entities", len(entities_to_put)) if commit: ndb.put_multi(entities_to_put) logging.info("Deleting %d keys", len(keys_to_delete)) if commit: ndb.delete_multi(keys_to_delete) logging.info("clear memcache") if commit: ndb.get_context().clear_cache() logging.info("Done!")