def createLagerbidrag(limits, scoutgroup, trooppersons, troopkey_key, bidrag):
from_date_time = datetime.strptime(bidrag.dateFrom + " 00:00",
DATE_TIME_FORMAT)
to_date_time = datetime.strptime(bidrag.dateTo + " 23:59",
DATE_TIME_FORMAT)
year = to_date_time.year
persons = []
validateLagetbidragInput(from_date_time, to_date_time, limits)
# Collect number of persons
for troopperson in trooppersons:
_add_person(troopperson.person, persons, year)
# Count number of days participating
for meeting in Meeting.query(Meeting.datetime >= from_date_time,
Meeting.datetime <= to_date_time,
Meeting.troop == troopkey_key).fetch():
for person in persons:
is_attending = person.person in meeting.attendingPersons
if is_attending:
person.days += 1
return createLagerbidragReport(limits, scoutgroup, persons, bidrag)
def createLagerbidragGroup(limits, scoutgroup, troops, bidrag):
from_date_time = datetime.strptime(bidrag.dateFrom + " 00:00",
DATE_TIME_FORMAT)
to_date_time = datetime.strptime(bidrag.dateTo + " 23:59",
DATE_TIME_FORMAT)
year = to_date_time.year
persons = []
person_days = {}
validateLagetbidragInput(from_date_time, to_date_time, limits)
for troop in troops:
# Count number of days participating
for meeting in Meeting.query(Meeting.datetime >= from_date_time,
Meeting.datetime <= to_date_time,
Meeting.troop == troop.key).fetch():
for person in meeting.attendingPersons:
if person not in person_days:
person_days[person] = 1
else:
person_days[person] += 1
# Collect number of persons
for person, days in person_days.iteritems():
_add_person(person, persons, year, days)
return createLagerbidragReport(limits, scoutgroup, persons, bidrag)
def meeting_date_count_deferred(start_cursor):
start_time = time.time()
# collect meetings per scoutgroup and day, count number of trooppersons and meeting attendees
meeting_date_counter_dict = memcache.get("meeting_date_counter_dict") # dict: datetime => number of attendee that day
# {"Scoutgroup X": {meetings:[{"date": 2020-01-01, pers:"23"}, {"date": 2020-01-02, pers:"33"}]}, "Scoutgroup Y": {meetings:[{"date": 2020-01-01, pers:"23"}, {"date": 2020-01-02, pers:"33"}]}}
there_is_more = true
while there_is_more:
sgs, start_cursor, there_is_more = ScoutGroup.query().fetch_page(page_size=1, start_cursor=start_cursor):
for sg in sgs:
logging.info("Updating:%s" % (semester.getname()))
for semester in Semester.query().fetch():
for troop_key in Troop.query(Troop.scoutgroup==sg.key, Troop.semester_key==semester.key).fetch(keys_only=True):
#troopPersonKeys = TroopPerson.query(TroopPerson.troop == troop.key).fetch(keys_only=True)
#numTroopPersons = len(TroopPerson)
for meeting in Meeting.query(Meeting.troop == troop_key).fetch():
attending_count = len(meeting.attendingPersons)
meeting_day = meeting.datetime.strftime('%Y-%m-%d')
if meeting_date_counter_dict.has_key(meeting_day):
attending_count += meeting_date_counter_dict[meeting_day]
meeting_date_counter_dict[meeting_day] = attending_count
if there_is_more:
memcache.delete("meeting_date_counter_dict")
memcache.add("meeting_date_counter_dict", meeting_date_counter_dict)
deferred.defer(meeting_date_count_deferred, start_cursor)
meeting_date_counter = [{"date" : day_key, "delt": meeting_date_counter_dict[day_key]} for day_key in sorted(meeting_date_counter_dict)]
logging.info("meeting_date_count update done")
memcache.delete("meeting_date_count")
memcache.add("meeting_date_count", meeting_date_counter)
memcache.delete("update_meeting_date_count_is_running")
def DeleteAllData():
entries = []
entries.extend(Person.query().fetch(keys_only=True))
entries.extend(Troop.query().fetch(keys_only=True))
entries.extend(ScoutGroup.query().fetch(keys_only=True))
entries.extend(Meeting.query().fetch(keys_only=True))
entries.extend(TroopPerson.query().fetch(keys_only=True))
entries.extend(Semester.query().fetch(keys_only=True))
entries.extend(TaskProgress.query().fetch(keys_only=True))
entries.extend(UserPrefs.query().fetch(keys_only=True))
ndb.delete_multi(entries)
ndb.get_context().clear_cache() # clear memcache
def show(troop_url, filetype):
troop = None
semester = None
scoutgroup = None
if troop_url is not None:
troop_key = ndb.Key(urlsafe=troop_url)
troop = troop_key.get()
semester = troop.semester_key.get()
scoutgroup = troop.scoutgroup.get()
meetings = Meeting.query(Meeting.troop == troop.key).fetch()
title = "Terminsprogram - " + semester.getname() + " - " + troop.getname(
) + " - " + scoutgroup.getname()
filename = title + "." + filetype
if filetype == "html":
return render_template('terminsprogram.html',
heading=title,
title=title,
meetings=meetings)
elif filetype == "xml":
xml = '<?xml version="1.0" encoding="utf-8"?>\r\n'
xml += '<terminsprogram semester="' + semester.getname(
) + '" troop="' + troop.getname(
) + '" scoutgroup="' + scoutgroup.getname() + '">\r\n'
xml += '<meetings>\r\n'
for meeting in meetings:
xml += '<meeting name="' + meeting.getname(
) + '" date="' + meeting.getdate() + '" time="' + meeting.gettime(
) + '" endtime="' + meeting.getendtime() + '"/>\r\n'
xml += '</meetings>\r\n'
xml += '</terminsprogram>\r\n'
response = make_response(xml)
response.headers['Content-Type'] = 'application/xml'
response.headers[
'Content-Disposition'] = 'attachment; filename=' + urllib.quote(
str(filename), safe='')
return response
elif filetype == "json":
json = "{"
json += '"semester":"' + semester.getname(
) + '", "troop":"' + troop.getname(
) + '", "scoutgroup=":"' + scoutgroup.getname() + '",\r\n'
json += '"meetings":[\r\n'
first = True
for meeting in meetings:
if not first: json += ",\r\n"
json += '{"name":"' + meeting.getname(
) + '", "date":"' + meeting.getdate(
) + '", "time":"' + meeting.gettime(
) + '", "endtime":"' + meeting.getendtime() + '"}'
first = False
json += ']\r\n'
json += "}"
response = make_response(json)
response.headers['Content-Type'] = 'text/json'
response.headers[
'Content-Disposition'] = 'attachment; filename=' + urllib.quote(
str(filename), safe='')
return response
else:
return "Unknown file format, html, xml or json is supported"
def scoutgroupsummary(sgroup_url):
user = UserPrefs.current()
if not user.canImport():
return "denied", 403
if sgroup_url is None:
return "missing group", 404
if user.activeSemester is None:
semester = Semester.getOrCreateCurrent()
else:
semester = user.activeSemester.get()
sgroup_key = ndb.Key(urlsafe=sgroup_url)
scoutgroup = sgroup_key.get()
breadcrumbs = [{'link':'/', 'text':'Hem'}]
baselink = "/groupsummary/" + sgroup_url
section_title = "Föreningsredovisning - " + scoutgroup.getname()
breadcrumbs.append({'link':baselink, 'text':section_title})
class Item():
age = 0
women = 0
womenMeetings = 0
men = 0
menMeetings = 0
def __init__(self, age, women=0, womenMeetings=0, men=0, menMeetings=0):
self.age = age
self.women = women
self.womenMeetings = womenMeetings
self.men = men
self.menMeetings = menMeetings
year = semester.year
women = 0
womenMeetings = 0
men = 0
menMeetings = 0
startage = 7
endage = 25
ages = [Item('0 - 6')]
ages.extend([Item(i) for i in range(startage, endage+1)])
ages.append(Item('26 - 64'))
ages.append(Item('65 -'))
leaders = [Item(u't.o.m. 25 år'), Item(u'över 25 år')]
boardmebers = [Item('')]
from_date_time = datetime.datetime.strptime(str(semester.year) + "-01-01 00:00", "%Y-%m-%d %H:%M")
to_date_time = datetime.datetime.strptime(str(semester.year) + "-12-31 00:00", "%Y-%m-%d %H:%M")
emails = []
for person in Person.query(Person.scoutgroup==sgroup_key).fetch():
if person.member_years is None or semester.year not in person.member_years:
continue
if person.email is not None and len(person.email) != 0 and person.email not in emails:
emails.append(person.email)
age = person.getyearsoldthisyear(year)
if scoutgroup.attendance_incl_hike:
number_of_meetings = Meeting.query(Meeting.attendingPersons==person.key,
Meeting.datetime >= from_date_time,
Meeting.datetime <= to_date_time).count()
else:
meetings = Meeting.query(Meeting.attendingPersons==person.key,
Meeting.datetime >= from_date_time,
Meeting.datetime <= to_date_time)
nr_all = meetings.count()
nr_hike_meetings = meetings.filter(Meeting.ishike == True).count()
number_of_meetings = nr_all - nr_hike_meetings
index = 0
if 7 <= age <= 25:
index = age-startage + 1
elif age < 7:
index = 0
elif 26 <= age <= 64:
index = endage - startage + 2
else:
index = endage - startage + 3
if person.isFemale():
women += 1
ages[index].women += 1
else:
men += 1
ages[index].men += 1
if number_of_meetings >= scoutgroup.attendance_min_year:
if person.isFemale():
womenMeetings += 1
ages[index].womenMeetings += 1
else:
menMeetings += 1
ages[index].menMeetings += 1
if person.isBoardMember():
if person.isFemale():
boardmebers[0].women += 1
else:
boardmebers[0].men += 1
if person.isLeader() and number_of_meetings > 1:
index = 0 if age <= 25 else 1
if person.isFemale():
leaders[index].women += 1
else:
leaders[index].men += 1
ages.append(Item("Totalt", women, womenMeetings, men, menMeetings))
return render_template('groupsummary.html', ages=ages, boardmebers=boardmebers, leaders=leaders,
breadcrumbs=breadcrumbs, emails=emails, year=semester.year,
min_nr_meetings=str(scoutgroup.attendance_min_year),
incl_hikes=scoutgroup.attendance_incl_hike)
def update_person_ids(commit, sgroup_key, start_cursor, stage, taskProgress):
start_time = time.time()
time_is_out = False
there_is_more = True
oldToNewDict = {}
# --- Stage 0 ---
if stage == 0:
while there_is_more:
if time.time() - start_time > max_time_seconds:
time_is_out = True
break # time out
person_keys, start_cursor, there_is_more = Person.query(Person.scoutgroup ==sgroup_key) \
.fetch_page(page_size=PAGE_SIZE, start_cursor=start_cursor, keys_only=True)
for person_key in person_keys:
if len(str(person_key.id())) < 12: # cannot be a personnr if shorter than 12 chars
taskProgress.info("Not updating id=%s" % (str(person_key.id())))
continue
person = person_key.get()
if person.version == 1 or person.version == 2:
taskProgress.info("Already updated id=%s" % (str(person_key.id())))
continue
if person.member_no is None:
taskProgress.warning("Cannot update, member_no is None id=%s, %s" % (str(person_key.id()), person.getname()))
continue
taskProgress.info("Updating Person: id=%s, pnr=%s, no=%s, %s" % (str(person.key.id()), person.personnr, str(person.member_no), person.getname()))
newPerson, pendingKeyChange = person.updateKey()
if newPerson is not None:
if commit and newPerson._dirty:
k = newPerson.put()
assert(k == newPerson.key)
if commit and person._dirty:
person.put()
if pendingKeyChange is not None:
if pendingKeyChange._dirty:
pendingKeyChange.put() # safe to put, to be able to test the rest of the method
if there_is_more:
return start_cursor, there_is_more, stage
else:
taskProgress.info("Updated all persons Persons")
stage = 1
start_cursor = None
there_is_more = True
return start_cursor, there_is_more, stage # restart to start fresh and let datastore catch up
taskProgress.info("Loading PendingPersonKeyChange records")
ppkc_more = True
ppkc_start_cursor = None
while ppkc_more:
ppkcs, ppkc_start_cursor, ppkc_more = PendingPersonKeyChange.query().fetch_page(page_size=1000, start_cursor=ppkc_start_cursor)
for ppkc in ppkcs:
oldToNewDict[ppkc.old_key] = ppkc.new_key
taskProgress.info("Loaded persons to convert %d" % (len(oldToNewDict)))
taskProgress.info("Number of persons to convert %d" % (len(oldToNewDict)))
# --- Stage 1 ---
if stage == 1:
num_meetings_updated = 0
troop_count = 0
there_is_more = True
taskProgress.info("Updating Meetings")
while there_is_more:
if time.time() - start_time > max_time_seconds:
return start_cursor, there_is_more, stage
troop_keys, start_cursor, there_is_more = Troop.query(Troop.scoutgroup == sgroup_key) \
.fetch_page(page_size=5, start_cursor=start_cursor, keys_only=True)
for troop_key in troop_keys:
troop_count += 1
meetings = Meeting.query(Meeting.troop == troop_key).fetch()
for meeting in meetings:
if meeting.uppdateOldPersonKeys(oldToNewDict):
num_meetings_updated += 1
if commit:
meeting.put()
if num_meetings_updated % 10 == 0:
taskProgress.info("Updated %d meetings, %d troop_count" % (num_meetings_updated, troop_count))
if there_is_more:
return start_cursor, there_is_more, stage
else:
taskProgress.info("Updated all meetings, time taken: %s s" % (str(time.time() - start_time)))
stage = 2
there_is_more = True
start_cursor = None
return start_cursor, there_is_more, stage
# --- Stage 2 ---
if stage == 2:
update_counter = 0
if start_cursor == None:
start_cursor = 0
taskProgress.info("Updating TroopPersons")
for index, (old_person_key, new_person_key) in enumerate(oldToNewDict.items()):
if index < start_cursor:
continue
for troopPerson in TroopPerson.query(TroopPerson.person==old_person_key).fetch():
troopPerson.person = new_person_key
update_counter += 1
if commit:
troopPerson.put()
if update_counter % 10 == 0:
taskProgress.info("Updated %d TroopPersons" % (update_counter))
if time.time() - start_time > max_time_seconds:
start_cursor = index
there_is_more = True
return start_cursor, there_is_more, stage
taskProgress.info("Updated all TroopPersons")
there_is_more = True
stage = 3
start_cursor = None
return start_cursor, there_is_more, stage # restart to start fresh and let datastore catch up
# --- Stage 3 ---
if stage == 3:
taskProgress.info("Deleting old persons")
if commit and len(oldToNewDict) != 0:
old_person_keys = [old_person_key for index, (old_person_key, new_person_key) in enumerate(oldToNewDict.items())]
ndb.delete_multi(old_person_keys)
taskProgress.info("Deleted %d old persons" % (len(old_person_keys)))
there_is_more = False
start_cursor = None
stage = 4
taskProgress.info("Done updating Person keys!")
return None, False, stage
def merge_scout_group(oldname, newname, commit, taskProgress, user, move_users, move_persons, move_troops, delete_sg, semester_id):
start_time = time.time()
oldsg = ScoutGroup.getbyname(oldname)
if oldsg is None:
raise RuntimeError("Old sg name:%s not found" % oldname)
newsg = ScoutGroup.getbyname(newname)
if newsg is None:
raise RuntimeError("New sg name:%s not found" % newname)
if not commit:
taskProgress.info("*** testmode ***")
keys_to_delete = []
entities_to_put_first = []
entities_to_put = []
taskProgress.info("termin: %s" % semester_id)
convertsemester = Semester.getbyId(semester_id)
if convertsemester is None:
taskProgress.error("termin: %s does not exist" % semester_id)
currentsemester = Semester.getOrCreateCurrent()
if move_users:
taskProgress.info("Update all users to the new scoutgroup")
for u in UserPrefs.query(UserPrefs.groupaccess == oldsg.key).fetch():
logging.info(" * * moving user for %s" % (u.getname()))
u.groupaccess = newsg.key
u.activeSemester = currentsemester.key
entities_to_put.append(u)
if move_persons:
taskProgress.info("Moving all persons to the new ScoutGroup:%s" % newsg.getname())
for oldp in Person.query(Person.scoutgroup == oldsg.key).fetch():
oldp.scoutgroup = newsg.key
entities_to_put.append(oldp)
if move_troops:
taskProgress.info("Move all troops to the new ScoutGroup:%s, semester: %s" % (newsg.getname(), convertsemester.getname()))
for oldt in Troop.query(Troop.scoutgroup == oldsg.key, Troop.semester_key == convertsemester.key).fetch():
taskProgress.info(" * found old troop for %s, semester=%s" % (str(oldt.key.id()), oldt.semester_key.get().getname()))
keys_to_delete.append(oldt.key)
newt = Troop.get_by_id(Troop.getid(oldt.scoutnetID, newsg.key, oldt.semester_key), use_memcache=True)
if newt is None:
taskProgress.info(" * * creating new troop for %s, semester=%s" % (str(oldt.key.id()), oldt.semester_key.get().getname()))
newt = Troop.create(oldt.name, oldt.scoutnetID, newsg.key, oldt.semester_key)
entities_to_put_first.append(newt) # put first to be able to reference it
else:
taskProgress.info(" * * already has new troop for %s, semester=%s" % (str(newt.key.id()), newt.semester_key.get().getname()))
taskProgress.info(" * Move all trooppersons to the new group")
for oldtp in TroopPerson.query(TroopPerson.troop == oldt.key).fetch():
keys_to_delete.append(oldtp.key)
newtp = TroopPerson.get_by_id(TroopPerson.getid(newt.key, oldtp.person), use_memcache=True)
if newtp is None:
logging.info(" * * creating new TroopPerson for %s:%s" % (newt.getname(), oldtp.getname()))
newtp = TroopPerson.create_or_update(newt.key, oldtp.person, oldtp.leader)
entities_to_put.append(newtp)
else:
logging.info(" * * already has TroopPerson for %s:%s" % (newt.getname(), oldtp.getname()))
taskProgress.info(" * Move all old meetings to the new troop")
for oldm in Meeting.query(Meeting.troop==oldt.key).fetch():
keys_to_delete.append(oldm.key)
newm = Meeting.get_by_id(Meeting.getId(oldm.datetime, newt.key), use_memcache=True)
if newm is None:
logging.info(" * * creating new Meeting for %s:%s" % (newt.getname(), oldm.datetime.strftime("%Y-%m-%d %H:%M")))
newm = Meeting(id=Meeting.getId(oldm.datetime, newt.key),
datetime=oldm.datetime,
name=oldm.name,
troop=newt.key,
duration=oldm.duration,
ishike=oldm.ishike)
newm.attendingPersons = oldm.attendingPersons
entities_to_put.append(newm)
else:
logging.info(" * * merging Meeting %s->%s :%s" % (oldm.getname(), newm.getname(), oldm.datetime.strftime("%Y-%m-%d %H:%M")))
need_to_put = False
if len(oldm.name) > len(newm.name): # take the longer name.
newm.name = oldm.name
need_to_put = True
for oldattendingpersonkey in oldm.attendingPersons:
if oldattendingpersonkey not in newm.attendingPersons:
newm.attendingPersons.append(oldattendingpersonkey)
need_to_put = True
if need_to_put:
entities_to_put.append(newm)
if delete_sg:
keys_to_delete.append(oldsg.key)
taskProgress.info("time before put: %s s" % str(time.time() - start_time))
logging.info("Putting %d entities first" % len(entities_to_put_first))
if commit: ndb.put_multi(entities_to_put_first)
logging.info("Putting %d entities" % len(entities_to_put))
if commit: ndb.put_multi(entities_to_put)
logging.info("Deleting %d keys" % len(keys_to_delete))
if commit: ndb.delete_multi(keys_to_delete)
logging.info("clear memcache")
if commit: ndb.get_context().clear_cache()
taskProgress.info("Done! time: %s s" % str(time.time() - start_time))
def merge_sg(oldname, newname, commit):
oldsg = ScoutGroup.getbyname(oldname)
if oldsg is None:
raise RuntimeError("Old sg name:%s not found" % oldname)
newsg = ScoutGroup.getbyname(newname)
if newsg is None:
raise RuntimeError("New sg name:%s not found" % newname)
if not commit: logging.info("*** testmode ***")
keys_to_delete = []
entities_to_put_first = []
entities_to_put = []
semester = Semester.getOrCreateCurrent()
keys_to_delete.append(oldsg.key)
logging.info("Update all users to the new scoutgroup")
for u in UserPrefs.query(UserPrefs.groupaccess == oldsg.key).fetch():
u.groupaccess = newsg.key
u.activeSemester = semester.key
entities_to_put.append(u)
logging.info("Moving all persons to the new ScoutGroup:%s", newsg.getname())
for oldp in Person.query(Person.scoutgroup == oldsg.key).fetch():
logging.info(" * Moving %s %s", oldp.getname(), oldp.personnr)
oldp.scoutgroup = newsg.key
entities_to_put.append(oldp)
logging.info("Move all troops to the new ScoutGroup:%s", newsg.getname())
for oldt in Troop.query(Troop.scoutgroup == oldsg.key).fetch():
logging.info(" * found old troop for %s, semester=%s", str(oldt.key.id()), oldt.semester_key.get().getname())
keys_to_delete.append(oldt.key)
newt = Troop.get_by_id(Troop.getid(oldt.scoutnetID, newsg.key, oldt.semester_key), use_memcache=True)
if newt is None:
logging.info(" * * creating new troop for %s, semester=%s", str(oldt.key.id()), oldt.semester_key.get().getname())
newt = Troop.create(oldt.name, oldt.scoutnetID, newsg.key, oldt.semester_key)
entities_to_put_first.append(newt) # put first to be able to reference it
else:
logging.info(" * * already has new troop for %s, semester=%s", str(newt.key.id()), newt.semester_key.get().getname())
logging.info(" * Move all trooppersons to the new group (it they don't already exist there)")
for oldtp in TroopPerson.query(TroopPerson.troop == oldt.key).fetch():
keys_to_delete.append(oldtp.key)
newtp = TroopPerson.get_by_id(TroopPerson.getid(newt.key, oldtp.person), use_memcache=True)
if newtp is None:
logging.info(" * * creating new TroopPerson for %s:%s", newt.getname(), oldtp.getname())
newtp = TroopPerson.create(newt.key, oldtp.person, oldtp.leader)
entities_to_put.append(newtp)
else:
logging.info(" * * already has TroopPerson for %s:%s", newt.getname(), oldtp.getname())
logging.info(" * Move all old meetings to the new troop")
for oldm in Meeting.query(Meeting.troop==oldt.key).fetch():
keys_to_delete.append(oldm.key)
newm = Meeting.get_by_id(Meeting.getId(oldm.datetime, newt.key), use_memcache=True)
if newm is None:
logging.info(" * * creating new Meeting for %s:%s", newt.getname(), oldm.datetime.strftime("%Y-%m-%d %H:%M"))
newm = Meeting.getOrCreate(newt.key, oldm.name, oldm.datetime, oldm.duration, oldm.ishike)
newm.attendingPersons = oldm.attendingPersons
entities_to_put.append(newm)
else:
logging.info(" * * already has Meeting for %s:%s", newt.getname(), newt.datetime.strftime("%Y-%m-%d %H:%M"))
logging.info("Putting %d entities first", len(entities_to_put_first))
if commit: ndb.put_multi(entities_to_put_first)
logging.info("Putting %d entities", len(entities_to_put))
if commit: ndb.put_multi(entities_to_put)
logging.info("Deleting %d keys", len(keys_to_delete))
if commit: ndb.delete_multi(keys_to_delete)
logging.info("clear memcache")
if commit: ndb.get_context().clear_cache()
logging.info("Done!")