示例#1
0
文件: stats.py 项目: Scouterna/skojjt
def meeting_date_count_deferred(start_cursor):
    start_time = time.time()
    # collect meetings per scoutgroup and day, count number of trooppersons and meeting attendees 
    meeting_date_counter_dict = memcache.get("meeting_date_counter_dict") # dict: datetime => number of attendee that day
    # {"Scoutgroup X": {meetings:[{"date": 2020-01-01, pers:"23"}, {"date": 2020-01-02, pers:"33"}]}, "Scoutgroup Y": {meetings:[{"date": 2020-01-01, pers:"23"}, {"date": 2020-01-02, pers:"33"}]}}
    there_is_more = true
    while there_is_more:
        sgs, start_cursor, there_is_more = ScoutGroup.query().fetch_page(page_size=1, start_cursor=start_cursor):
        for sg in sgs:
            logging.info("Updating:%s" % (semester.getname()))
            for semester in Semester.query().fetch():
                for troop_key in Troop.query(Troop.scoutgroup==sg.key, Troop.semester_key==semester.key).fetch(keys_only=True):
                    #troopPersonKeys = TroopPerson.query(TroopPerson.troop == troop.key).fetch(keys_only=True)
                    #numTroopPersons = len(TroopPerson)
                    for meeting in Meeting.query(Meeting.troop == troop_key).fetch():
                        attending_count = len(meeting.attendingPersons)
                        meeting_day = meeting.datetime.strftime('%Y-%m-%d')

                        if meeting_date_counter_dict.has_key(meeting_day):
                            attending_count += meeting_date_counter_dict[meeting_day]
                        
                        meeting_date_counter_dict[meeting_day] = attending_count
            
            if there_is_more:
                memcache.delete("meeting_date_counter_dict")
                memcache.add("meeting_date_counter_dict", meeting_date_counter_dict)
                deferred.defer(meeting_date_count_deferred, start_cursor)


    meeting_date_counter = [{"date" : day_key, "delt": meeting_date_counter_dict[day_key]} for day_key in sorted(meeting_date_counter_dict)]
    
    logging.info("meeting_date_count update done")
    memcache.delete("meeting_date_count")
    memcache.add("meeting_date_count", meeting_date_counter)
    memcache.delete("update_meeting_date_count_is_running")
示例#2
0
文件: admin.py 项目: Scouterna/skojjt
def dosettroopsemester():
    semester_key = Semester.getOrCreateCurrent().key
    troops = Troop.query().fetch()
    for troop in troops:
        #if troop.semester_key != semester_key:
        troop.semester_key = semester_key
        logging.info("updating semester for: %s", troop.getname())
        troop.put()
示例#3
0
def DeleteAllData():
    entries = []
    entries.extend(Person.query().fetch(keys_only=True))
    entries.extend(Troop.query().fetch(keys_only=True))
    entries.extend(ScoutGroup.query().fetch(keys_only=True))
    entries.extend(Meeting.query().fetch(keys_only=True))
    entries.extend(TroopPerson.query().fetch(keys_only=True))
    entries.extend(Semester.query().fetch(keys_only=True))
    entries.extend(TaskProgress.query().fetch(keys_only=True))
    entries.extend(UserPrefs.query().fetch(keys_only=True))
    ndb.delete_multi(entries)
    ndb.get_context().clear_cache()  # clear memcache
示例#4
0
文件: admin.py 项目: Scouterna/skojjt
def update_person_ids(commit, sgroup_key, start_cursor, stage, taskProgress):
    start_time = time.time()
    time_is_out = False
    there_is_more = True
    oldToNewDict = {}

    # --- Stage 0 ---
    if stage == 0:
        while there_is_more:

            if time.time() - start_time > max_time_seconds:
                time_is_out = True
                break # time out

            person_keys, start_cursor, there_is_more = Person.query(Person.scoutgroup ==sgroup_key) \
                                                        .fetch_page(page_size=PAGE_SIZE, start_cursor=start_cursor, keys_only=True)
            for person_key in person_keys:
                if len(str(person_key.id())) < 12: # cannot be a personnr if shorter than 12 chars
                    taskProgress.info("Not updating id=%s" % (str(person_key.id())))
                    continue
                person = person_key.get()
                if person.version == 1 or person.version == 2:
                    taskProgress.info("Already updated id=%s" % (str(person_key.id())))
                    continue
                if person.member_no is None:
                    taskProgress.warning("Cannot update, member_no is None id=%s, %s" % (str(person_key.id()), person.getname()))
                    continue

                taskProgress.info("Updating Person: id=%s, pnr=%s, no=%s, %s" % (str(person.key.id()), person.personnr, str(person.member_no), person.getname()))

                newPerson, pendingKeyChange = person.updateKey()

                if newPerson is not None:
                    if commit and newPerson._dirty:
                        k = newPerson.put()
                        assert(k == newPerson.key)
                if commit and person._dirty:
                    person.put()
                if pendingKeyChange is not None:
                    if pendingKeyChange._dirty:
                        pendingKeyChange.put() # safe to put, to be able to test the rest of the method

        if there_is_more:
            return start_cursor, there_is_more, stage
        else:
            taskProgress.info("Updated all persons Persons")
            stage = 1
            start_cursor = None
            there_is_more = True
            return start_cursor, there_is_more, stage # restart to start fresh and let datastore catch up


    taskProgress.info("Loading PendingPersonKeyChange records")
    ppkc_more = True
    ppkc_start_cursor = None
    while ppkc_more:
        ppkcs, ppkc_start_cursor, ppkc_more = PendingPersonKeyChange.query().fetch_page(page_size=1000, start_cursor=ppkc_start_cursor)
        for ppkc in ppkcs:
            oldToNewDict[ppkc.old_key] = ppkc.new_key
        
        taskProgress.info("Loaded persons to convert %d" % (len(oldToNewDict)))

    taskProgress.info("Number of persons to convert %d" % (len(oldToNewDict)))

    # --- Stage 1 ---
    if stage == 1:
        num_meetings_updated = 0
        troop_count = 0
        there_is_more = True
        taskProgress.info("Updating Meetings")
        while there_is_more:
            if time.time() - start_time > max_time_seconds:
                return start_cursor, there_is_more, stage

            troop_keys, start_cursor, there_is_more = Troop.query(Troop.scoutgroup == sgroup_key) \
                                                    .fetch_page(page_size=5, start_cursor=start_cursor, keys_only=True)
            for troop_key in troop_keys:
                troop_count += 1
                meetings = Meeting.query(Meeting.troop == troop_key).fetch()
                for meeting in meetings:
                    if meeting.uppdateOldPersonKeys(oldToNewDict):
                        num_meetings_updated += 1
                        if commit:
                            meeting.put()

            if num_meetings_updated % 10 == 0:
                taskProgress.info("Updated %d meetings, %d troop_count" % (num_meetings_updated, troop_count))

        if there_is_more:
            return start_cursor, there_is_more, stage
        else:
            taskProgress.info("Updated all meetings, time taken: %s s" % (str(time.time() - start_time)))
            stage = 2
            there_is_more = True
            start_cursor = None
            return start_cursor, there_is_more, stage

    # --- Stage 2 ---
    if stage == 2:
        update_counter = 0
        if start_cursor == None:
            start_cursor = 0

        taskProgress.info("Updating TroopPersons")
        for index, (old_person_key, new_person_key) in enumerate(oldToNewDict.items()):
            if index < start_cursor:
                continue

            for troopPerson in TroopPerson.query(TroopPerson.person==old_person_key).fetch():
                troopPerson.person = new_person_key
                update_counter += 1
                if commit:
                    troopPerson.put()
                if update_counter % 10 == 0:
                    taskProgress.info("Updated %d TroopPersons" % (update_counter))

            if time.time() - start_time > max_time_seconds:
                start_cursor = index
                there_is_more = True
                return start_cursor, there_is_more, stage

        taskProgress.info("Updated all TroopPersons")
        there_is_more = True
        stage = 3
        start_cursor = None
        return start_cursor, there_is_more, stage # restart to start fresh and let datastore catch up

    # --- Stage 3 ---
    if stage == 3:
        taskProgress.info("Deleting old persons")
        if commit and len(oldToNewDict) != 0:
            old_person_keys = [old_person_key for index, (old_person_key, new_person_key) in enumerate(oldToNewDict.items())]
            ndb.delete_multi(old_person_keys)
            taskProgress.info("Deleted %d old persons" % (len(old_person_keys)))

        there_is_more = False
        start_cursor = None
        stage = 4

    taskProgress.info("Done updating Person keys!")
    return None, False, stage
示例#5
0
文件: admin.py 项目: Scouterna/skojjt
def merge_scout_group(oldname, newname, commit, taskProgress, user, move_users, move_persons, move_troops, delete_sg, semester_id):
    start_time = time.time()
    oldsg = ScoutGroup.getbyname(oldname)
    if oldsg is None:
        raise RuntimeError("Old sg name:%s not found" % oldname)

    newsg = ScoutGroup.getbyname(newname)
    if newsg is None:
        raise RuntimeError("New sg name:%s not found" % newname)

    if not commit:
        taskProgress.info("*** testmode ***")

    keys_to_delete = []
    entities_to_put_first = []
    entities_to_put = []

    taskProgress.info("termin: %s" % semester_id)
    
    convertsemester = Semester.getbyId(semester_id)
    if convertsemester is None:
        taskProgress.error("termin: %s does not exist" % semester_id)

    currentsemester = Semester.getOrCreateCurrent()

    if move_users:
        taskProgress.info("Update all users to the new scoutgroup")
        for u in UserPrefs.query(UserPrefs.groupaccess == oldsg.key).fetch():
            logging.info(" * * moving user for %s" % (u.getname()))
            u.groupaccess = newsg.key
            u.activeSemester = currentsemester.key
            entities_to_put.append(u)

    if move_persons:
        taskProgress.info("Moving all persons to the new ScoutGroup:%s" % newsg.getname())
        for oldp in Person.query(Person.scoutgroup == oldsg.key).fetch():
            oldp.scoutgroup = newsg.key
            entities_to_put.append(oldp)

    if move_troops:
        taskProgress.info("Move all troops to the new ScoutGroup:%s, semester: %s" % (newsg.getname(), convertsemester.getname()))
        for oldt in Troop.query(Troop.scoutgroup == oldsg.key, Troop.semester_key == convertsemester.key).fetch():
            taskProgress.info(" * found old troop for %s, semester=%s" % (str(oldt.key.id()), oldt.semester_key.get().getname()))
            keys_to_delete.append(oldt.key)
            newt = Troop.get_by_id(Troop.getid(oldt.scoutnetID, newsg.key, oldt.semester_key), use_memcache=True)
            if newt is None:
                taskProgress.info(" * * creating new troop for %s, semester=%s" % (str(oldt.key.id()), oldt.semester_key.get().getname()))
                newt = Troop.create(oldt.name, oldt.scoutnetID, newsg.key, oldt.semester_key)
                entities_to_put_first.append(newt) # put first to be able to reference it
            else:
                taskProgress.info(" * * already has new troop for %s, semester=%s" % (str(newt.key.id()), newt.semester_key.get().getname()))

            taskProgress.info(" * Move all trooppersons to the new group")
            for oldtp in TroopPerson.query(TroopPerson.troop == oldt.key).fetch():
                keys_to_delete.append(oldtp.key)
                newtp = TroopPerson.get_by_id(TroopPerson.getid(newt.key, oldtp.person), use_memcache=True)
                if newtp is None:
                    logging.info(" * * creating new TroopPerson for %s:%s" % (newt.getname(), oldtp.getname()))
                    newtp = TroopPerson.create_or_update(newt.key, oldtp.person, oldtp.leader)
                    entities_to_put.append(newtp)
                else:
                    logging.info(" * * already has TroopPerson for %s:%s" % (newt.getname(), oldtp.getname()))

            taskProgress.info(" * Move all old meetings to the new troop")
            for oldm in Meeting.query(Meeting.troop==oldt.key).fetch():
                keys_to_delete.append(oldm.key)
                newm = Meeting.get_by_id(Meeting.getId(oldm.datetime, newt.key), use_memcache=True)
                if newm is None:
                    logging.info(" * * creating new Meeting for %s:%s" % (newt.getname(), oldm.datetime.strftime("%Y-%m-%d %H:%M")))
                    newm = Meeting(id=Meeting.getId(oldm.datetime, newt.key),
                                        datetime=oldm.datetime,
                                        name=oldm.name,
                                        troop=newt.key,
                                        duration=oldm.duration,
                                        ishike=oldm.ishike)
                    newm.attendingPersons = oldm.attendingPersons
                    entities_to_put.append(newm)
                else:
                    logging.info(" * * merging Meeting %s->%s :%s" % (oldm.getname(), newm.getname(), oldm.datetime.strftime("%Y-%m-%d %H:%M")))
                    need_to_put = False
                    if len(oldm.name) > len(newm.name): # take the longer name.
                        newm.name = oldm.name
                        need_to_put = True
                    
                    for oldattendingpersonkey in oldm.attendingPersons:
                        if oldattendingpersonkey not in newm.attendingPersons:
                            newm.attendingPersons.append(oldattendingpersonkey)
                            need_to_put = True
                    
                    if need_to_put:
                        entities_to_put.append(newm)

        if delete_sg:
            keys_to_delete.append(oldsg.key)

    taskProgress.info("time before put: %s s" % str(time.time() - start_time))

    logging.info("Putting %d entities first" % len(entities_to_put_first))
    if commit: ndb.put_multi(entities_to_put_first)
    logging.info("Putting %d entities" % len(entities_to_put))
    if commit: ndb.put_multi(entities_to_put)
    logging.info("Deleting %d keys" % len(keys_to_delete))
    if commit: ndb.delete_multi(keys_to_delete)
    logging.info("clear memcache")
    if commit: ndb.get_context().clear_cache()

    taskProgress.info("Done! time: %s s" % str(time.time() - start_time))
示例#6
0
文件: main.py 项目: kesse/skojjt
def merge_sg(oldname, newname, commit):
    oldsg = ScoutGroup.getbyname(oldname)
    if oldsg is None:
        raise RuntimeError("Old sg name:%s not found" % oldname)

    newsg = ScoutGroup.getbyname(newname)
    if newsg is None:
        raise RuntimeError("New sg name:%s not found" % newname)

    if not commit: logging.info("*** testmode ***")

    keys_to_delete = []
    entities_to_put_first = []
    entities_to_put = []
    semester = Semester.getOrCreateCurrent()

    keys_to_delete.append(oldsg.key)

    logging.info("Update all users to the new scoutgroup")
    for u in UserPrefs.query(UserPrefs.groupaccess == oldsg.key).fetch():
        u.groupaccess = newsg.key
        u.activeSemester = semester.key
        entities_to_put.append(u)

    logging.info("Moving all persons to the new ScoutGroup:%s", newsg.getname())
    for oldp in Person.query(Person.scoutgroup == oldsg.key).fetch():
        logging.info(" * Moving %s %s", oldp.getname(), oldp.personnr)
        oldp.scoutgroup = newsg.key
        entities_to_put.append(oldp)

    logging.info("Move all troops to the new ScoutGroup:%s", newsg.getname())
    for oldt in Troop.query(Troop.scoutgroup == oldsg.key).fetch():
        logging.info(" * found old troop for %s, semester=%s", str(oldt.key.id()), oldt.semester_key.get().getname())
        keys_to_delete.append(oldt.key)
        newt = Troop.get_by_id(Troop.getid(oldt.scoutnetID, newsg.key, oldt.semester_key), use_memcache=True)
        if newt is None:
            logging.info(" * * creating new troop for %s, semester=%s", str(oldt.key.id()), oldt.semester_key.get().getname())
            newt = Troop.create(oldt.name, oldt.scoutnetID, newsg.key, oldt.semester_key)
            entities_to_put_first.append(newt) # put first to be able to reference it
        else:
            logging.info(" * * already has new troop for %s, semester=%s", str(newt.key.id()), newt.semester_key.get().getname())

        logging.info(" * Move all trooppersons to the new group (it they don't already exist there)")
        for oldtp in TroopPerson.query(TroopPerson.troop == oldt.key).fetch():
            keys_to_delete.append(oldtp.key)
            newtp = TroopPerson.get_by_id(TroopPerson.getid(newt.key, oldtp.person), use_memcache=True)
            if newtp is None:
                logging.info(" * * creating new TroopPerson for %s:%s", newt.getname(), oldtp.getname())
                newtp = TroopPerson.create(newt.key, oldtp.person, oldtp.leader)
                entities_to_put.append(newtp)
            else:
                logging.info(" * * already has TroopPerson for %s:%s", newt.getname(), oldtp.getname())

        logging.info(" * Move all old meetings to the new troop")
        for oldm in Meeting.query(Meeting.troop==oldt.key).fetch():
            keys_to_delete.append(oldm.key)
            newm = Meeting.get_by_id(Meeting.getId(oldm.datetime, newt.key), use_memcache=True)
            if newm is None:
                logging.info(" * * creating new Meeting for %s:%s", newt.getname(), oldm.datetime.strftime("%Y-%m-%d %H:%M"))
                newm = Meeting.getOrCreate(newt.key, oldm.name, oldm.datetime, oldm.duration, oldm.ishike)
                newm.attendingPersons = oldm.attendingPersons
                entities_to_put.append(newm)
            else:
                logging.info(" * * already has Meeting for %s:%s", newt.getname(), newt.datetime.strftime("%Y-%m-%d %H:%M"))

    logging.info("Putting %d entities first", len(entities_to_put_first))
    if commit: ndb.put_multi(entities_to_put_first)
    logging.info("Putting %d entities", len(entities_to_put))
    if commit: ndb.put_multi(entities_to_put)
    logging.info("Deleting %d keys", len(keys_to_delete))
    if commit: ndb.delete_multi(keys_to_delete)
    logging.info("clear memcache")
    if commit: ndb.get_context().clear_cache()
    logging.info("Done!")