def main():
parser = ArgumentParser()
parser.add_argument('interest')
parser.add_argument('--nomembers',
default=False,
action='store_const',
const=True,
help='use --nomembers to skip members')
parser.add_argument('--nomanagers',
default=False,
action='store_const',
const=True,
help='use --nomanagers to skip managers')
args = parser.parse_args()
scriptdir = dirname(__file__)
# two levels up
scriptfolder = dirname(dirname(scriptdir))
configdir = join(scriptfolder, 'config')
memberconfigfile = "members.cfg"
memberconfigpath = join(configdir, memberconfigfile)
userconfigfile = "users.cfg"
userconfigpath = join(configdir, userconfigfile)
# create app and get configuration
# use this order so members.cfg overrrides users.cfg
configfiles = [userconfigpath, memberconfigpath]
app = create_app(Development(configfiles), configfiles)
# set up database
db.init_app(app)
# set up app and request contexts so that taskdetails works
# https://flask.palletsprojects.com/en/1.1.x/testing/
with app.app_context(), app.test_request_context():
# turn on logging
setlogging()
# g has to be set within app context
g.interest = args.interest
membertasks = []
taskdetails.open()
for row in taskdetails.rows:
membertask = taskdetails.dte.get_response_data(row)
# add user record
localuserid, taskid = taskdetails.getids(row.id)
membertask['User'] = localuser2user(localuserid)
membertask['Task'] = Task.query.filter_by(id=taskid).one()
membertasks.append(membertask)
# create member based data structure
mem2tasks = {}
for membertask in membertasks:
mem2tasks.setdefault(membertask['User'].email, {'tasks': []})
mem2tasks[membertask['User'].email]['tasks'].append(membertask)
for member in mem2tasks:
mem2tasks[member]['tasks'].sort(key=lambda t: t['order'])
# default is from interest, may be overridden below, based on emailtemplate configuration
fromlist = localinterest().from_email
# allows for debugging of each section separately
if not args.nomembers:
emailtemplate = EmailTemplate.query.filter_by(
templatename='member-email', interest=localinterest()).one()
template = Template(emailtemplate.template)
subject = emailtemplate.subject
if emailtemplate.from_email:
fromlist = emailtemplate.from_email
refurl = url_for('admin.taskchecklist',
interest=g.interest,
_external=True)
for emailaddr in mem2tasks:
# only send email if there are tasks overdue or upcoming
sendforstatus = [STATUS_EXPIRES_SOON, STATUS_OVERDUE]
if len([
t for t in mem2tasks[emailaddr]['tasks']
if t['status'] in sendforstatus
]) > 0:
html = template.render(**mem2tasks[emailaddr],
statuses=sendforstatus,
refurl=refurl)
tolist = emailaddr
cclist = None
sendmail(subject, fromlist, tolist, html, ccaddr=cclist)
# allows for debugging of each section separately
if not args.nomanagers:
# what groups are each member a part of?
member2groups = {}
for memberlocal in LocalUser.query.filter_by(
active=True, interest=localinterest()).all():
memberglobal = localuser2user(memberlocal)
member2groups[memberglobal.email] = {
'worker': memberlocal,
'taskgroups': set()
}
# drill down to get all taskgroups the member is responsible for
for position in positions_active(memberlocal, date.today()):
get_position_taskgroups(
position,
member2groups[memberglobal.email]['taskgroups'])
for taskgroup in memberlocal.taskgroups:
get_taskgroup_taskgroups(
taskgroup,
member2groups[memberglobal.email]['taskgroups'])
# get list of responsible managers
responsibility = {}
positions = Position.query.filter_by(
interest=localinterest()).all()
for position in positions:
positiontasks = set()
positionworkers = set()
theseemailgroups = set(position.emailgroups)
for workeremail in member2groups:
# add worker if the worker's taskgroups intersect with these email groups
if theseemailgroups & member2groups[workeremail][
'taskgroups']:
positionworkers |= {
member2groups[workeremail]['worker']
}
for emailgroup in position.emailgroups:
get_taskgroup_tasks(emailgroup, positiontasks)
# only set responsibility if this position has management for some groups
if position.emailgroups:
for manager in members_active(position, date.today()):
manageruser = localuser2user(manager)
responsibility.setdefault(manageruser.email, {
'tasks': set(),
'workers': set()
})
responsibility[
manageruser.email]['tasks'] |= positiontasks
responsibility[
manageruser.email]['workers'] |= positionworkers
# set up template engine
emailtemplate = EmailTemplate.query.filter_by(
templatename='leader-email', interest=localinterest()).one()
template = Template(emailtemplate.template)
subject = emailtemplate.subject
if emailtemplate.from_email:
fromlist = emailtemplate.from_email
refurl = url_for('admin.taskdetails',
interest=g.interest,
_external=True)
# loop through responsible managers, setting up their email
for manager in responsibility:
manager2members = {'members': []}
# need to convert to ids which are given by taskdetails
for positionworker in responsibility[manager]['workers']:
resptasks = [
taskdetails.setid(positionworker.id, t.id)
for t in responsibility[manager]['tasks']
]
positionuser = localuser2user(positionworker)
thesetasks = [
t for t in mem2tasks[positionuser.email]['tasks']
if t['rowid'] in resptasks
and t['status'] in [STATUS_OVERDUE]
]
if thesetasks:
manager2members['members'].append({
'name':
positionuser.name,
'tasks':
thesetasks
})
# only send if something to send
if manager2members['members']:
html = template.render(**manager2members, refurl=refurl)
tolist = manager
cclist = None
sendmail(subject, fromlist, tolist, html, ccaddr=cclist)
def main():
descr = '''
Update racing team info volunteer records from csv file
'''
parser = ArgumentParser(description=descr)
parser.add_argument('inputfile',
help='csv file with input records',
default=None)
args = parser.parse_args()
scriptdir = dirname(__file__)
# two levels up
scriptfolder = dirname(dirname(scriptdir))
configdir = join(scriptfolder, 'config')
memberconfigfile = "members.cfg"
memberconfigpath = join(configdir, memberconfigfile)
userconfigfile = "users.cfg"
userconfigpath = join(configdir, userconfigfile)
# create app and get configuration
# use this order so members.cfg overrrides users.cfg
configfiles = [userconfigpath, memberconfigpath]
app = create_app(Development(configfiles), configfiles)
# set up database
db.init_app(app)
# determine input file encoding
with open(args.inputfile, 'rb') as binaryfile:
rawdata = binaryfile.read()
detected = detect(rawdata)
# translate type from old format to new
applntype = {
'Returning Racing Team Member': 'renewal',
'New Racing Team Member': 'new',
}
# need app context, open input file
with app.app_context(), open(args.inputfile,
'r',
encoding=detected['encoding'],
newline='',
errors='replace') as IN:
# turn on logging
setlogging()
# trick local interest stuff
g.interest = 'fsrc'
# initialize database tables from input file
infile = DictReader(IN)
for row in infile:
# this pulls timezone information off of record timestamp, formatted like 'Sun Feb 25 2018 14:07:17 GMT-0500 (EST)'
timestampasc = ' '.join(row['time'].split(' ')[:-2])
timestamp = tstamp.asc2dt(timestampasc)
# if we already have received an application for this name at this timestamp, skip it else we'll get duplicates
applnrec = RacingTeamApplication.query.filter_by(
name=row['name'],
logtime=timestamp,
**localinterest_query_params()).one_or_none()
if applnrec: continue
# at least one record doesn't have a date of birth
if not row['dob']:
app.logger.warning(
f"racingteam_appln_init: skipping {row['name']} {row['race1-name']} {row[f'race1-date']}"
)
continue
# if we've gotten here, we need to add application and result records
dob = isodate.asc2dt(row['dob']).date()
applnrec = RacingTeamApplication(
interest=localinterest(),
logtime=timestamp,
name=row['name'],
type=applntype[row['applntype']],
comments=row['comments'],
dateofbirth=dob,
email=row['email'],
gender=row['gender'].upper()[0],
)
db.session.add(applnrec)
for race in ['race1', 'race2']:
# originally, new members were only asked for one race
# detect this condition and skip this result -- this should only happen for race2
if not row[f'{race}-date']: continue
# handle case where age grade was not calculated properly
# this was due to deficiency in the original script, so these should be early entries
# it's not worth adding the complexity to fix this data at this point
try:
agegrade = float(row[f'{race}-agegrade']),
agegrade = row[f'{race}-agegrade']
except ValueError:
agegrade = None
# calculate age
racedate = isodate.asc2dt(row[f'{race}-date']).date()
thisage = age(racedate, dob)
# add result
resultrec = RacingTeamResult(
interest=localinterest(),
application=applnrec,
eventdate=racedate,
eventname=row[f'{race}-name'],
age=thisage,
agegrade=agegrade,
distance=row[f'{race}-distance'],
units=row[f'{race}-units'],
location=row[f'{race}-location'],
url=row[f'{race}-resultslink'],
time=row[f'{race}-time'],
)
db.session.add(resultrec)
db.session.commit()
def main():
descr = '''
Update racing team info volunteer records from csv file
'''
parser = ArgumentParser(description=descr)
parser.add_argument('inputfile', help='csv file with input records', default=None)
args = parser.parse_args()
scriptdir = dirname(__file__)
# two levels up
scriptfolder = dirname(dirname(scriptdir))
configdir = join(scriptfolder, 'config')
memberconfigfile = "members.cfg"
memberconfigpath = join(configdir, memberconfigfile)
userconfigfile = "users.cfg"
userconfigpath = join(configdir, userconfigfile)
# create app and get configuration
# use this order so members.cfg overrrides users.cfg
configfiles = [userconfigpath, memberconfigpath]
app = create_app(Development(configfiles), configfiles)
# set up database
db.init_app(app)
# determine input file encoding
with open(args.inputfile, 'rb') as binaryfile:
rawdata = binaryfile.read()
detected = detect(rawdata)
# need app context, open input file
with app.app_context(), open(args.inputfile, 'r', encoding=detected['encoding'], newline='', errors='replace') as IN:
# turn on logging
setlogging()
# trick local interest stuff
g.interest = 'fsrc'
# initialize database tables from input file
infile = DictReader(IN)
for row in infile:
# first check if racing team member exists
localuser = LocalUser.query.filter_by(name=row['name'], **localinterest_query_params()).one_or_none()
member = RacingTeamMember.query.filter_by(localuser=localuser, **localinterest_query_params()).one_or_none() if localuser else None
if not member: continue
# this pulls timezone information off of timestamp, formatted like 'Sun Feb 25 2018 14:07:17 GMT-0500 (EST)'
timestampasc = ' '.join(row['timestamp'].split(' ')[:-2])
timestamp = tstamp.asc2dt(timestampasc)
# if we already have received an info record for this member at this timestamp, skip it else we'll get duplicates
inforec = RacingTeamInfo.query.filter_by(member=member, logtime=timestamp).one_or_none()
if inforec: continue
# if we've gotten here, we need to add info and volunteer records
inforec = RacingTeamInfo(interest=localinterest(), member=member, logtime=timestamp)
db.session.add(inforec)
volrec = RacingTeamVolunteer(
interest=localinterest(),
info=inforec,
eventdate = isodate.asc2dt(row['eventdate']).date(),
eventname = row['eventname'],
hours = row['hours'],
comment = row['comments'],
)
db.session.add(volrec)
db.session.commit()
def update(interest, membershipfile):
"""update member, membership tables, from membershipfile if supplied, or from service based on interest"""
thislogger = getLogger('members.cli')
if debug:
thislogger.setLevel(DEBUG)
else:
thislogger.setLevel(INFO)
thislogger.propagate = True
# set local interest
g.interest = interest
linterest = localinterest()
# assume update will complete ok
tableupdatetime = TableUpdateTime.query.filter_by(
interest=linterest, tablename='member').one_or_none()
if not tableupdatetime:
tableupdatetime = TableUpdateTime(interest=linterest,
tablename='member')
db.session.add(tableupdatetime)
tableupdatetime.lastchecked = datetime.today()
# normal case is download from RunSignUp service
if not membershipfile:
# get, check club id
club_id = linterest.service_id
if not (linterest.club_service == 'runsignup' and club_id):
raise ParameterError(
'interest Club Service must be runsignup, and Service ID must be defined'
)
# transform: membership "file" format from RunSignUp API
xform = Transform(
{
'MemberID':
lambda mem: mem['user']['user_id'],
'MembershipID':
'membership_id',
'MembershipType':
'club_membership_level_name',
'FamilyName':
lambda mem: mem['user']['last_name'],
'GivenName':
lambda mem: mem['user']['first_name'],
'MiddleName':
lambda mem: mem['user']['middle_name']
if mem['user']['middle_name'] else '',
'Gender':
lambda mem: 'Female'
if mem['user']['gender'] == 'F' else 'Male',
'DOB':
lambda mem: mem['user']['dob'],
'City':
lambda mem: mem['user']['address']['city'],
'State':
lambda mem: mem['user']['address']['state'],
'Email':
lambda mem: mem['user']['email']
if 'email' in mem['user'] else '',
'PrimaryMember':
'primary_member',
'JoinDate':
'membership_start',
'ExpirationDate':
'membership_end',
'LastModified':
'last_modified',
},
sourceattr=False, # source and target are dicts
targetattr=False)
rsu = RunSignUp(key=current_app.config['RSU_KEY'],
secret=current_app.config['RSU_SECRET'],
debug=debug)
def doxform(ms):
membership = {}
xform.transform(ms, membership)
return membership
with rsu:
# get current and future members from RunSignUp, and put into common format
rawmemberships = rsu.members(club_id, current_members_only='F')
currfuturememberships = [
m for m in rawmemberships
if m['membership_end'] >= datetime.today().date().isoformat()
]
memberships = [doxform(ms) for ms in currfuturememberships]
# membershipfile supplied
else:
with open(membershipfile, 'r') as _MF:
MF = DictReader(_MF)
# memberships already in common format
memberships = [ms for ms in MF]
# sort memberships by member (family_name, given_name, gender, dob), expiration_date
memberships.sort(key=lambda m: (m['FamilyName'], m['GivenName'], m[
'Gender'], m['DOB'], m['ExpirationDate']))
# set up member, membership transforms to create db records
# transform: member record from membership "file" format
memxform = Transform(
{
'family_name':
'FamilyName',
'given_name':
'GivenName',
'middle_name':
'MiddleName',
'gender':
'Gender',
'svc_member_id':
'MemberID',
'dob':
lambda m: isodate.asc2dt(m['DOB']).date(),
'hometown':
lambda m: f'{m["City"]}, {m["State"]}'
if 'City' in m and 'State' in m else '',
'email':
'Email',
'start_date':
lambda m: isodate.asc2dt(m['JoinDate']).date(),
'end_date':
lambda m: isodate.asc2dt(m['ExpirationDate']).date(),
},
sourceattr=False,
targetattr=True)
# transform: update member record from membership record
memupdate = Transform(
{
'svc_member_id': 'svc_member_id',
'hometown': 'hometown',
'email': 'email',
},
sourceattr=True,
targetattr=True)
# transform: membership record from membership "file" format
mshipxform = Transform(
{
'svc_member_id':
'MemberID',
'svc_membership_id':
'MembershipID',
'membershiptype':
'MembershipType',
'hometown':
lambda m: f'{m["City"]}, {m["State"]}'
if 'City' in m and 'State' in m else '',
'email':
'Email',
'start_date':
lambda m: isodate.asc2dt(m['JoinDate']).date(),
'end_date':
lambda m: isodate.asc2dt(m['ExpirationDate']).date(),
'primary':
lambda m: m['PrimaryMember'].lower() == 't' or m['PrimaryMember'].
lower() == 'yes',
'last_modified':
lambda m: rsudt.asc2dt(m['LastModified']),
},
sourceattr=False,
targetattr=True)
# insert member, membership records
for m in memberships:
# need MembershipId to be string for comparison with database key
m['MembershipID'] = str(m['MembershipID'])
filternamedob = and_(Member.family_name == m['FamilyName'],
Member.given_name == m['GivenName'],
Member.gender == m['Gender'],
Member.dob == isodate.asc2dt(m['DOB']))
# func.binary forces case sensitive comparison. see https://stackoverflow.com/a/31788828/799921
filtermemberid = Member.svc_member_id == func.binary(m['MemberID'])
filtermember = or_(filternamedob, filtermemberid)
# get all the member records for this member
# note there may currently be more than one member record, as the memberships may be discontiguous
thesemembers = SortedList(key=lambda member: member.end_date)
thesemembers.update(Member.query.filter(filtermember).all())
# if member doesn't exist, create member and membership records
if len(thesemembers) == 0:
thismember = Member(interest=localinterest())
memxform.transform(m, thismember)
db.session.add(thismember)
# flush so thismember can be referenced in thismship, and can be found in later processing
db.session.flush()
thesemembers.add(thismember)
thismship = Membership(interest=localinterest(), member=thismember)
mshipxform.transform(m, thismship)
db.session.add(thismship)
# flush so thismship can be found in later processing
db.session.flush()
# if there are already some memberships for this member, merge with this membership (m)
else:
# dbmships is keyed by svc_membership_id, sorted by end_date
# NOTE: membership id is unique only within a member -- be careful if the use of dbmships changes
# to include multiple members
dbmships = ItemSortedDict(lambda k, v: v.end_date)
for thismember in thesemembers:
for mship in thismember.memberships:
dbmships[mship.svc_membership_id] = mship
# add membership if not already there for this member
mshipid = m['MembershipID']
if mshipid not in dbmships:
newmship = True
thismship = Membership(interest=localinterest())
db.session.add(thismship)
# flush so thismship can be found in later processing
db.session.flush()
# update existing membership
else:
newmship = False
thismship = dbmships[mshipid]
# merge the new membership record into the database record
mshipxform.transform(m, thismship)
# add new membership to data structure
if newmship:
dbmships[thismship.svc_membership_id] = thismship
# need list view for some processing
dbmships_keys = dbmships.keys()
# check for overlaps
for thisndx in range(1, len(dbmships_keys)):
prevmship = dbmships[dbmships_keys[thisndx - 1]]
thismship = dbmships[dbmships_keys[thisndx]]
if thismship.start_date <= prevmship.end_date:
oldstart = thismship.start_date
newstart = prevmship.end_date + timedelta(1)
oldstartasc = isodate.dt2asc(oldstart)
newstartasc = isodate.dt2asc(newstart)
endasc = isodate.dt2asc(thismship.end_date)
memberkey = f'{m["FamilyName"]},{m["GivenName"]},{m["DOB"]}'
thislogger.warn(
f'overlap detected for {memberkey}: end={endasc} was start={oldstartasc} now start={newstartasc}'
)
thismship.start_date = newstart
# update appropriate member record(s), favoring earlier member records
# NOTE: membership hometown, email get copied into appropriate member records;
# since mship list is sorted, last one remains
for mshipid in dbmships_keys:
mship = dbmships[mshipid]
for nextmndx in range(len(thesemembers)):
thismember = thesemembers[nextmndx]
lastmember = thesemembers[nextmndx -
1] if nextmndx != 0 else None
# TODO: use Transform for these next four entries
# corner case: someone changed their birthdate
thismember.dob = isodate.asc2dt(m['DOB']).date()
# prefer last name found
thismember.given_name = m['GivenName']
thismember.family_name = m['FamilyName']
thismember.middle_name = m['MiddleName'] if m[
'MiddleName'] else ''
# mship causes new member record before this one
# or after end of thesemembers
# or wholy between thesemembers
if (mship.end_date + timedelta(1) < thismember.start_date
or (nextmndx == len(thesemembers) - 1)
and mship.start_date >
thismember.end_date + timedelta(1)
or lastmember and mship.start_date >
lastmember.end_date + timedelta(1)
and mship.end_date < thismember.start_date):
newmember = Member(interest=localinterest())
# flush so thismember can be referenced in mship, and can be found in later processing
db.session.flush()
memxform.transform(m, newmember)
mship.member = newmember
break
# mship extends this member record from the beginning
if mship.end_date + timedelta(1) == thismember.start_date:
thismember.start_date = mship.start_date
mship.member = thismember
memupdate.transform(mship, thismember)
break
# mship extends this member from the end
if mship.start_date == thismember.end_date + timedelta(1):
thismember.end_date = mship.end_date
mship.member = thismember
memupdate.transform(mship, thismember)
break
# mship end date was changed
if (mship.start_date >= thismember.start_date
and mship.start_date <= thismember.end_date
and mship.end_date != thismember.end_date):
thismember.end_date = mship.end_date
mship.member = thismember
memupdate.transform(mship, thismember)
break
# mship start date was changed
if (mship.end_date >= thismember.start_date
and mship.end_date <= thismember.end_date
and mship.start_date != thismember.start_date):
thismember.start_date = mship.start_date
mship.member = thismember
memupdate.transform(mship, thismember)
break
# mship wholly contained within this member
if mship.start_date >= thismember.start_date and mship.end_date <= thismember.end_date:
mship.member = thismember
memupdate.transform(mship, thismember)
break
# delete unused member records
delmembers = []
for mndx in range(len(thesemembers)):
thismember = thesemembers[mndx]
if len(thismember.memberships) == 0:
delmembers.append(thismember)
for delmember in delmembers:
db.session.delete(delmember)
thesemembers.remove(delmember)
if len(delmembers) > 0:
db.session.flush()
# merge member records as appropriate
thisndx = 0
delmembers = []
for nextmndx in range(1, len(thesemembers)):
thismember = thesemembers[thisndx]
nextmember = thesemembers[nextmndx]
if thismember.end_date + timedelta(1) == nextmember.start_date:
for mship in nextmember.memberships:
mship.member = thismember
delmembers.append(nextmember)
else:
thisndx = nextmndx
for delmember in delmembers:
db.session.delete(delmember)
if len(delmembers) > 0:
db.session.flush()
# save statistics file
groupfolder = join(current_app.config['APP_FILE_FOLDER'], interest)
if not exists(groupfolder):
mkdir(groupfolder, mode=0o770)
statspath = join(groupfolder, current_app.config['APP_STATS_FILENAME'])
analyzemembership(statsfile=statspath)
# make sure we remember everything we did
db.session.commit()