def members2file(club_id, mapping, key, secret, outfile=None):
#----------------------------------------------------------------------
'''
convert members added per day (ordyears) to membercount per day in json format
:param club_id: RunSignUp club id
:param mapping: OrderedDict {'outfield1':'infield1', 'outfield2':outfunction(memberrec), ...}
:param outfile: optional output file
:param key: RunSignUp key
:param secret: RunSignUp secret
:rtype: lines from output file
'''
# pull in memberfile
rsu = RunSignUp(key=key, secret=secret)
rsu.open()
members = rsu.members(club_id)
rsu.close()
# analyze mapping for outfields
xform = Transform(mapping, sourceattr=False, targetattr=False)
outfields = mapping.keys()
# create writeable list, csv file
memberlist = wlist()
cmemberlist = DictWriter(memberlist, outfields)
cmemberlist.writeheader()
for thismember in members:
outrow = {}
xform.transform(thismember, outrow)
cmemberlist.writerow(outrow)
# write file if desired
if outfile:
with open(outfile,'wb') as out:
out.writelines(memberlist)
return memberlist
from . import version
from loutilities.transform import Transform
from loutilities.timeu import asctime, age
from datetime import date
from collections import defaultdict, OrderedDict
# time stuff
tymd = asctime('%Y-%m-%d')
# transform DETAILS file produced by scoretility Results Analysis
xform = Transform(
{
'name' : 'runnername',
'gender' : 'gender',
'age' : lambda result: age(date.today(), tymd.asc2dt(result['dob'])),
'distmiles' : 'distmiles',
'ag' : lambda result: int(float(result['agpercent'])),
'year' : lambda result: tymd.asc2dt(result['racedate']).year
},
sourceattr=False,
targetattr=True)
# # from https://gist.github.com/shenwei356/71dcc393ec4143f3447d
# # from: http://stackoverflow.com/questions/651794/whats-the-best-way-to-initialize-a-dict-of-dicts-in-python
# #----------------------------------------------------------------------
# def ddict():
# #----------------------------------------------------------------------
# return defaultdict(ddict)
#######################################################################
class Result():
def updatemembercache(club_id,
membercachefilename,
key=None,
secret=None,
email=None,
password=None,
debug=False):
#----------------------------------------------------------------------
if debug:
# set up debug logging
thislogger.setLevel(logging.DEBUG)
thislogger.propagate = True
else:
# error logging
thislogger.setLevel(logging.ERROR)
thislogger.propagate = True
# set up access to RunSignUp
rsu = RunSignUp(key=key,
secret=secret,
email=email,
password=password,
debug=debug)
rsu.open()
# transform from RunSignUp to membercache format
xform = Transform(
{
'MemberID':
lambda mem: mem['user']['user_id'],
'MembershipID':
'membership_id',
'MembershipType':
'club_membership_level_name',
'FamilyName':
lambda mem: mem['user']['last_name'],
'GivenName':
lambda mem: mem['user']['first_name'],
'MiddleName':
lambda mem: mem['user']['middle_name'],
'Gender':
lambda mem: 'Female' if mem['user']['gender'] == 'F' else 'Male',
'DOB':
lambda mem: mem['user']['dob'],
'Email':
lambda mem: mem['user']['email'] if 'email' in mem['user'] else '',
'PrimaryMember':
'primary_member',
'JoinDate':
'membership_start',
'ExpirationDate':
'membership_end',
'LastModified':
'last_modified',
},
sourceattr=False, # source and target are dicts
targetattr=False)
# members maintains the current cache through this processing {memberkey: [memberrec, ...]}
# currmemberrecs maintains the records for current members as of today {memberkey: memberrec}
members = {}
currmemberrecs = {}
# need today's date, in same sortable date format as data coming from RunSignUp
dt = asctime('%Y-%m-%d')
today = dt.dt2asc(datetime.now())
# construct key from member cache record
def getmemberkey(memberrec):
lastname = memberrec['FamilyName']
firstname = memberrec['GivenName']
dob = memberrec['DOB']
memberkey = '{},{},{}'.format(lastname, firstname, dob)
return memberkey
# add record to cache, return key
def add2cache(memberrec):
memberkey = getmemberkey(memberrec)
members.setdefault(memberkey, [])
# replace any records having same expiration date
recordlist = [
mr for mr in members[memberkey]
if mr['ExpirationDate'] != memberrec['ExpirationDate']
] + [memberrec]
members[memberkey] = recordlist
# keep list sorted
sortby = 'ExpirationDate'
members[memberkey].sort(lambda a, b: cmp(a[sortby], b[sortby]))
# remove any overlaps
for i in range(1, len(members[memberkey])):
lastrec = members[memberkey][i - 1]
thisrec = members[memberkey][i]
# if there's an overlap, change join date to expiration date + 1 day
if thisrec['JoinDate'] <= lastrec['ExpirationDate']:
exp = thisrec['ExpirationDate']
oldstart = thisrec['JoinDate']
newstart = dt.dt2asc(
dt.asc2dt(lastrec['ExpirationDate']) + timedelta(1))
thislogger.error(
'overlap detected: {} end={} was start={} now start={}'.
format(memberkey, exp, oldstart, newstart))
thisrec['JoinDate'] = newstart
return memberkey
# test if in cache
def incache(memberrec):
memberkey = getmemberkey(memberrec)
if memberkey not in members:
cachedmember = False
elif memberrec['ExpirationDate'] in [
m['ExpirationDate'] for m in members[memberkey]
]:
cachedmember = True
else:
cachedmember = False
return cachedmember
# lock cache update during execution
rlock = RLock()
with rlock:
# track duration of update
starttime = datetime.now()
# import current cache
# records in cache are organized in members dict with 'last,first,dob' key
# within is list of memberships ordered by expiration date
with open(membercachefilename, 'rb') as memfile:
# members maintains the current cache through this processing
# currmemberrecs maintains the records for current members as of today
cachedmembers = DictReader(memfile)
for memberrec in cachedmembers:
memberkey = add2cache(memberrec)
# current member?
if memberrec['JoinDate'] <= today and memberrec[
'ExpirationDate'] >= today:
# member should only be in current members once
if memberkey in currmemberrecs:
thislogger.error(
'member duplicated in cache: {}'.format(memberkey))
# regardless add this record to current members
currmemberrecs[memberkey] = memberrec
# get current members from RunSignUp, transforming each to cache format
rsumembers = rsu.members(club_id)
rsucurrmembers = []
for rsumember in rsumembers:
memberrec = {}
xform.transform(rsumember, memberrec)
rsucurrmembers.append(memberrec)
# add new member records to cache
# remove known (not new) member records from currmemberrecs
# after loop currmemberrecs should contain only deleted member records
for memberrec in rsucurrmembers:
# remember if was incache before we add
currmember = incache(memberrec)
# this will replace record with same ExpirationDate
# this allows admin updated RunSignUp data to be captured in cache
memberkey = add2cache(memberrec)
# remove member records we knew about already
if currmember:
del currmemberrecs[memberkey]
# remove member records for deleted members
for memberkey in currmemberrecs:
removedrec = currmemberrecs[memberkey]
memberkey = getmemberkey(removedrec)
members[memberkey] = [
mr for mr in members[memberkey] if mr != removedrec
]
thislogger.debug(
'membership removed from cache: {}'.format(removedrec))
# recreate cache file
# start with temporary file
# sort members keys for ease of debugging
cachedir = dirname(abspath(membercachefilename))
sortedmembers = sorted(members.keys())
with NamedTemporaryFile(mode='wb',
suffix='.rsucache',
delete=False,
dir=cachedir) as tempcache:
tempmembercachefilename = tempcache.name
cachehdr = 'MemberID,MembershipID,MembershipType,FamilyName,GivenName,MiddleName,Gender,DOB,Email,PrimaryMember,JoinDate,ExpirationDate,LastModified'.split(
',')
cache = DictWriter(tempcache, cachehdr)
cache.writeheader()
for memberkey in sortedmembers:
for memberrec in members[memberkey]:
cache.writerow(memberrec)
# set mode of temp file to be same as current cache file (see https://stackoverflow.com/questions/5337070/how-can-i-get-a-files-permission-mask)
cachemode = stat(membercachefilename).st_mode & 0777
chmod(tempmembercachefilename, cachemode)
# now overwrite the previous version of the membercachefile with the new membercachefile
try:
# atomic operation in Linux
rename(tempmembercachefilename, membercachefilename)
# should only happen under windows
except OSError:
remove(membercachefilename)
rename(tempmembercachefilename, membercachefilename)
# track duration of update
finishtime = datetime.now()
thislogger.debug('updatemembercache() duration={}'.format(finishtime -
starttime))
# release access
rsu.close()
# let caller know the current members, in rsu api format
return rsumembers
# read routes and import into database
## first delete all the files
interestfolders = glob(join(app.config['APP_FILE_FOLDER'], '*'))
for interestfolder in interestfolders:
rmtree(interestfolder, ignore_errors=True)
## now read the routes
routeswb = load_workbook(routesfile, read_only=True)
routessheet = routeswb['routes']
routesrows = routessheet.iter_rows(min_row=1, values_only=True)
# set up to transform routes file to Route database record
routeshdr = 'name,distance,start location,latlng,surface,elevation gain,fileid,map,description,active'.split(',')
routedbfields = 'name,distance,start_location,latlng,surface,elevation_gain,gpx_file_id,map,description,active'.split(',')
routemapping = dict(zip(routedbfields, routeshdr))
routex = Transform(routemapping, sourceattr=False, targetattr=True)
# set up to transform path sheet to path csv file
pathxlhdr = 'lat,lng,orig ele,res,ele,cumdist(km),inserted'.split(',')
pathcsvhdr = 'lat,lng,orig_ele,res,ele,cumdist_km,inserted'.split(',')
pathmapping = dict(zip(pathcsvhdr, pathxlhdr))
pathx = Transform(pathmapping, sourceattr=False, targetattr=False)
fsrc = Interest.query.filter_by(interest='fsrc').one()
routeshdr = next(routesrows) # pulls header
routescols = routeshdr.index(None)
for in_route in routesrows:
# seems to be more rows than those which have data, so break out at first empty row
if in_route[0] == None: break
thisroute = dict(zip(routeshdr[:routescols], in_route[:routescols]))
def updatemembercache(club_id, membercachefilename, key=None, secret=None, email=None, password=None, debug=False):
#----------------------------------------------------------------------
if debug:
# set up debug logging
thislogger.setLevel(logging.DEBUG)
thislogger.propagate = True
else:
# error logging
thislogger.setLevel(logging.ERROR)
thislogger.propagate = True
# set up access to RunSignUp
rsu = RunSignUp(key=key, secret=secret, email=email, password=password, debug=debug)
rsu.open()
# transform from RunSignUp to membercache format
xform = Transform( {
'MemberID' : lambda mem: mem['user']['user_id'],
'MembershipID' : 'membership_id',
'MembershipType' : 'club_membership_level_name',
'FamilyName' : lambda mem: mem['user']['last_name'],
'GivenName' : lambda mem: mem['user']['first_name'],
'MiddleName' : lambda mem: mem['user']['middle_name'],
'Gender' : lambda mem: 'Female' if mem['user']['gender'] == 'F' else 'Male',
'DOB' : lambda mem: mem['user']['dob'],
'Email' : lambda mem: mem['user']['email'] if 'email' in mem['user'] else '',
'PrimaryMember' : 'primary_member',
'JoinDate' : 'membership_start',
'ExpirationDate' : 'membership_end',
'LastModified' : 'last_modified',
},
sourceattr=False, # source and target are dicts
targetattr=False
)
# members maintains the current cache through this processing {memberkey: [memberrec, ...]}
# currmemberrecs maintains the records for current members as of today {memberkey: memberrec}
members = {}
currmemberrecs = {}
# need today's date, in same sortable date format as data coming from RunSignUp
dt = asctime('%Y-%m-%d')
today = dt.dt2asc(datetime.now())
# construct key from member cache record
def getmemberkey(memberrec):
lastname = memberrec['FamilyName']
firstname = memberrec['GivenName']
dob = memberrec['DOB']
memberkey = '{},{},{}'.format(lastname, firstname, dob)
return memberkey
# add record to cache, return key
def add2cache(memberrec):
memberkey = getmemberkey(memberrec)
members.setdefault(memberkey,[])
# replace any records having same expiration date
recordlist = [mr for mr in members[memberkey] if mr['ExpirationDate'] != memberrec['ExpirationDate']] + [memberrec]
members[memberkey] = recordlist
# keep list sorted
sortby = 'ExpirationDate'
members[memberkey].sort(lambda a,b: cmp(a[sortby],b[sortby]))
# remove any overlaps
for i in range(1, len(members[memberkey])):
lastrec = members[memberkey][i-1]
thisrec = members[memberkey][i]
# if there's an overlap, change join date to expiration date + 1 day
if thisrec['JoinDate'] <= lastrec['ExpirationDate']:
exp = thisrec['ExpirationDate']
oldstart = thisrec['JoinDate']
newstart = dt.dt2asc( dt.asc2dt(lastrec['ExpirationDate']) + timedelta(1) )
thislogger.error('overlap detected: {} end={} was start={} now start={}'.format(memberkey, exp, oldstart, newstart))
thisrec['JoinDate'] = newstart
return memberkey
# test if in cache
def incache(memberrec):
memberkey = getmemberkey(memberrec)
if memberkey not in members:
cachedmember = False
elif memberrec['ExpirationDate'] in [m['ExpirationDate'] for m in members[memberkey]]:
cachedmember = True
else:
cachedmember = False
return cachedmember
# lock cache update during execution
rlock = RLock()
with rlock:
# track duration of update
starttime = datetime.now()
# import current cache
# records in cache are organized in members dict with 'last,first,dob' key
# within is list of memberships ordered by expiration date
with open(membercachefilename, 'rb') as memfile:
# members maintains the current cache through this processing
# currmemberrecs maintains the records for current members as of today
cachedmembers = DictReader(memfile)
for memberrec in cachedmembers:
memberkey = add2cache(memberrec)
# current member?
if memberrec['JoinDate'] <= today and memberrec['ExpirationDate'] >= today:
# member should only be in current members once
if memberkey in currmemberrecs:
thislogger.error( 'member duplicated in cache: {}'.format(memberkey) )
# regardless add this record to current members
currmemberrecs[memberkey] = memberrec
# get current members from RunSignUp, transforming each to cache format
rsumembers = rsu.members(club_id)
rsucurrmembers = []
for rsumember in rsumembers:
memberrec = {}
xform.transform(rsumember, memberrec)
rsucurrmembers.append(memberrec)
# add new member records to cache
# remove known (not new) member records from currmemberrecs
# after loop currmemberrecs should contain only deleted member records
for memberrec in rsucurrmembers:
# remember if was incache before we add
currmember = incache(memberrec)
# this will replace record with same ExpirationDate
# this allows admin updated RunSignUp data to be captured in cache
memberkey = add2cache(memberrec)
# remove member records we knew about already
# if not there, skip. probably replaced record in cache
if currmember:
try:
del currmemberrecs[memberkey]
except KeyError:
pass
# remove member records for deleted members
for memberkey in currmemberrecs:
removedrec = currmemberrecs[memberkey]
memberkey = getmemberkey(removedrec)
members[memberkey] = [mr for mr in members[memberkey] if mr != removedrec]
thislogger.debug('membership removed from cache: {}'.format(removedrec))
# recreate cache file
# start with temporary file
# sort members keys for ease of debugging
cachedir = dirname(abspath(membercachefilename))
sortedmembers = sorted(members.keys())
with NamedTemporaryFile(mode='wb', suffix='.rsucache', delete=False, dir=cachedir) as tempcache:
tempmembercachefilename = tempcache.name
cachehdr = 'MemberID,MembershipID,MembershipType,FamilyName,GivenName,MiddleName,Gender,DOB,Email,PrimaryMember,JoinDate,ExpirationDate,LastModified'.split(',')
cache = DictWriter(tempcache, cachehdr)
cache.writeheader()
for memberkey in sortedmembers:
for memberrec in members[memberkey]:
cache.writerow(memberrec)
# set mode of temp file to be same as current cache file (see https://stackoverflow.com/questions/5337070/how-can-i-get-a-files-permission-mask)
cachemode = stat(membercachefilename).st_mode & 0777
chmod(tempmembercachefilename, cachemode)
# now overwrite the previous version of the membercachefile with the new membercachefile
try:
# atomic operation in Linux
rename(tempmembercachefilename, membercachefilename)
# should only happen under windows
except OSError:
remove(membercachefilename)
rename(tempmembercachefilename, membercachefilename)
# track duration of update
finishtime = datetime.now()
thislogger.debug( 'updatemembercache() duration={}'.format(finishtime-starttime) )
# release access
rsu.close()
# let caller know the current members, in rsu api format
return rsumembers
def importmembers(configfile, debug=False, stats=False):
#----------------------------------------------------------------------
'''
import member data to mailchimp
configfile must have the following
[mailchimp]
RSU_CLUB: <runsignup club_id>
RSU_KEY: <key from runsignup partnership>
RSU_SECRET: <secret from runsignup partnership>
MC_KEY: <api key from MailChimp>
MC_LIST: <name of list of interest>
MC_GROUPNAMES: groupname1,groupname2,...
MC_SHADOWCATEGORY: <name of shadowcategory>
* shadowcategory groups are used to show desired inclusion but the group itself under other categories can be toggled by subscriber
* this is used to prevent the recipient from being added back to the group against their wishes
* if a groupname is not also under shadowcategory, it is only ticked if the subscriber was not present in the list prior to import
* this category's group names include all of the group names which are reserved for members
MC_CURRMEMBERGROUP: <name of group which is set for current members>
MC_PASTMEMBERGROUP: <name of group which is set for current and past members>
:param configfile: name of configuration file
:param debug: set to True for debug output
:param stats: set to True for stats output (INFO)
'''
# set up logging
thislogger.propagate = True
if debug:
# set up debug logging
thislogger.setLevel(logging.DEBUG)
elif stats:
# INFO logging
thislogger.setLevel(logging.INFO)
else:
# WARNING logging
thislogger.setLevel(logging.WARNING)
# load configuration
rsuconfig = getitems(configfile, 'runsignup')
mcconfig = getitems(configfile, 'mailchimp')
club_id = rsuconfig['RSU_CLUB']
rsukey = rsuconfig['RSU_KEY']
rsusecret = rsuconfig['RSU_SECRET']
mckey = mcconfig['MC_KEY']
mclist = mcconfig['MC_LIST']
mcgroupnames = mcconfig['MC_GROUPNAMES'].split(',')
mcshadowcategory = mcconfig['MC_SHADOWCATEGORY']
mcpastmembergroupname = mcconfig['MC_PASTMEMBERGROUP']
mccurrmembergroupname = mcconfig['MC_CURRMEMBERGROUP']
mctimeout = float(mcconfig['MC_TIMEOUT'])
# use Transform to simplify RunSignUp format
xform = Transform( {
'last' : lambda mem: mem['user']['last_name'],
'first' : lambda mem: mem['user']['first_name'],
'email' : lambda mem: mem['user']['email'] if 'email' in mem['user'] else '',
'primary' : lambda mem: mem['primary_member'] == 'T',
'start' : 'membership_start',
'end' : 'membership_end',
'modified' : 'last_modified',
},
# source and target are dicts, not objects
sourceattr=False,
targetattr=False
)
# download current member list from RunSignUp
# get current members from RunSignUp, transforming each to local format
# only save one member per email address, primary member preferred
rsu = RunSignUp(key=rsukey, secret=rsusecret, debug=debug)
rsu.open()
rsumembers = rsu.members(club_id)
rsucurrmembers = {}
for rsumember in rsumembers:
memberrec = {}
xform.transform(rsumember, memberrec)
memberkey = memberrec['email'].lower()
# only save if there's an email address
# the primary member takes precedence, but if different email for nonprimary members save those as well
if memberkey and (memberrec['primary'] or memberkey not in rsucurrmembers):
rsucurrmembers[memberkey] = memberrec
rsu.close()
# It's important not to add someone back to a group which they've decided not to receive
# emails from. For this reason, a membergroup is defined with the same group names as
# the real groups the user is interested in, for those groups which don't make up the whole
# list.
# download categories / groups from MailChimp
client = MailChimp(mc_api=mckey, timeout=mctimeout)
lists = client.lists.all(get_all=True, fields="lists.name,lists.id")
list_id = [lst['id'] for lst in lists['lists'] if lst['name'] == mclist][0]
categories = client.lists.interest_categories.all(list_id=list_id,fields="categories.title,categories.id")
# groups are for anyone, shadowgroups are for members only
groups = {}
shadowgroups = {}
# for debugging
allgroups = {}
for category in categories['categories']:
mcgroups = client.lists.interest_categories.interests.all(list_id=list_id,category_id=category['id'],fields="interests.name,interests.id")
for group in mcgroups['interests']:
# save for debug
allgroups[group['id']] = '{} / {}'.format(category['title'], group['name'])
# special group to track past members
if group['name'] == mcpastmembergroupname:
mcpastmembergroup = group['id']
# and current members
elif group['name'] == mccurrmembergroupname:
mccurrmembergroup = group['id']
# newly found members are enrolled in all groups
elif category['title'] != mcshadowcategory:
groups[group['name']] = group['id']
# shadowgroups is used to remember the state of member's only groups for previous members
# if a member's membership has expired they must be removed from any group(s) which have the same name as
# those within the shadowgroup(s) (associated groups)
# additionally if any nonmembers are found, they must be removed from the associated groups
# this last bit can happen if someone who is not a member tries to enroll in a members only group
else:
shadowgroups[group['name']] = group['id']
# set up specific groups for mc api
mcapi = Obj()
# members new to the list get all the groups
mcapi.newmember = { id : True for id in groups.values() + shadowgroups.values() + [mccurrmembergroup] + [mcpastmembergroup]}
# previous members who lapsed get the member groups disabled
mcapi.nonmember = { id : False for id in [groups[gname] for gname in groups.keys() if gname in shadowgroups] + [mccurrmembergroup] }
# members groups set to True, for mcapi.unsubscribed merge
mcapi.member = { id : True for id in shadowgroups.values() + [groups[gname] for gname in groups.keys() if gname in shadowgroups] + [mccurrmembergroup] + [mcpastmembergroup]}
# unsubscribed members who previously were not past members get member groups turned on and 'other' groups turned off
mcapi.unsubscribed = merge_dicts (mcapi.member, { id:False for id in [groups[gname] for gname in groups.keys() if gname not in shadowgroups] })
# retrieve all members of this mailchimp list
# key these into dict by id (md5 has of lower case email address)
tmpmcmembers = client.lists.members.all(list_id=list_id, get_all=True, fields='members.id,members.email_address,members.status,members.merge_fields,members.interests')
mcmembers = {}
for mcmember in tmpmcmembers['members']:
mcmembers[mcmember['id']] = mcmember
# collect some stats
stat = Stat(['addedtolist', 'newmemberunsubscribed', 'newmember', 'pastmember',
'nonmember', 'memberunsubscribedskipped', 'membercleanedskipped',
'mailchimperror'])
# loop through club members
# if club member is in mailchimp
# make sure shadowgroups are set (but don't change groups as these may have been adjusted by club member)
# don't change subscribed status
# pop off mcmembers as we want to deal with the leftovers later
# if club member is not already in mailchimp
# add assuming all groups (groups + shadowgroups)
for memberkey in rsucurrmembers:
clubmember = rsucurrmembers[memberkey]
mcmemberid = mcid(clubmember['email'])
thislogger.debug( 'processing {} {}'.format(clubmember['email'], mcmemberid) )
# if club member is in mailchimp
if mcmemberid in mcmembers:
mcmember = mcmembers.pop(mcmemberid)
# check if any changes are required
# change required if current member not set
if not mcmember['interests'][mccurrmembergroup]:
# if not past member, just set the needful
if not mcmember['interests'][mcpastmembergroup]:
# if subscribed, all groups are set
if mcmember['status'] == 'subscribed':
client.lists.members.update(list_id=list_id, subscriber_hash=mcmemberid, data={'interests' : mcapi.newmember})
stat.newmember += 1
# if unsubscribed, subscribe them to member stuff, but remove everything else
elif mcmember['status'] == 'unsubscribed':
try:
client.lists.members.update(list_id=list_id, subscriber_hash=mcmemberid, data={'interests' : mcapi.unsubscribed, 'status' : 'subscribed'})
stat.newmemberunsubscribed += 1
# MailChimp won't let us resubscribe this member
except MailChimpError as e:
thislogger.info('member unsubscribed, skipped: {}'.format(clubmember['email']))
stat.memberunsubscribedskipped += 1
# other statuses are skipped
else:
thislogger.info('member cleaned, skipped: {}'.format(clubmember['email']))
stat.membercleanedskipped += 1;
# past member, recall what they had set before for the member stuff
else:
pastmemberinterests = merge_dicts({ groups[gname] : mcmember['interests'][shadowgroups[gname]] for gname in shadowgroups.keys() },
{ mccurrmembergroup : True })
client.lists.members.update(list_id=list_id, subscriber_hash=mcmemberid, data={'interests' : pastmemberinterests})
stat.pastmember += 1
# if club member is missing from mailchimp
else:
try:
client.lists.members.create(list_id=list_id,
data={
'email_address' : clubmember['email'],
'merge_fields' : {'FNAME' : clubmember['first'], 'LNAME' : clubmember['last'] },
'interests' : mcapi.newmember,
'status' : 'subscribed'
})
stat.addedtolist += 1
except MailChimpError as e:
ed = e.args[0]
thislogger.warning('MailChimpError {} for {}: {}'.format(ed['title'], clubmember['email'], ed['detail']))
stat.mailchimperror += 1
# at this point, mcmembers have only those enrollees who are not in the club
# loop through each of these and make sure club only interests are removed
for mcmemberid in mcmembers:
mcmember = mcmembers[mcmemberid]
# change required if current member set
if mcmember['interests'][mccurrmembergroup]:
# save member interests for later if they rejoin
memberinterests = {shadowgroups[gname]:mcmember['interests'][groups[gname]] for gname in shadowgroups}
client.lists.members.update(list_id=list_id, subscriber_hash=mcmemberid, data={'interests' : merge_dicts(mcapi.nonmember, memberinterests)})
stat.nonmember += 1
# log stats
thislogger.info ( stat )
def __init__(self, servicename, serviceaccessor, xservice2norm):
#----------------------------------------------------------------------
self.servicename = servicename
self.serviceaccessor = serviceaccessor
self.service2norm = Transform(xservice2norm, sourceattr=True, targetattr=True)
class StoreServiceResults():
########################################################################
'''
store results retrieved from a service, using service's file access class
note serviceentryid should be set only if this id increases with newer results
xservice2norm: mapping from service access attribute to a normalized data structure
must include all RaceResult attributes except
club_id, runnerid, raceid, seriesid
place and points attributes may also be omitted
must include additional keys used for Runner, Race lookup:
runnername, dob, gender, racename, raceloc, date, distmiles, serviceentryid
:param servicename: name of service
:param serviceaccessor: instance of ServiceResultFile
:param xservice2norm: {'normattr_n':'serviceattr_n', 'normattr_m':f(servicerow), ...}
'''
#----------------------------------------------------------------------
def __init__(self, servicename, serviceaccessor, xservice2norm):
#----------------------------------------------------------------------
self.servicename = servicename
self.serviceaccessor = serviceaccessor
self.service2norm = Transform(xservice2norm, sourceattr=True, targetattr=True)
#----------------------------------------------------------------------
def get_count(self, filename):
#----------------------------------------------------------------------
'''
return the length of the service accessor file
:param filename: name of the file
:rtype: number of lines in the file
'''
self.serviceaccessor.open(filename)
numlines = self.serviceaccessor.count()
self.serviceaccessor.close()
return numlines
#----------------------------------------------------------------------
def storeresults(self, thistask, status, club_id, filename):
#----------------------------------------------------------------------
'''
create service accessor and open file
get location if known
loop through all results in accessor file, and store in database
close file
caller needs to `db.session.commit()` the changes
:param thistask: this is required for task thistask.update_state()
:param status: status for updating front end
:param club_id: identifies club for which results are to be stored
:param filename: name of csv file which contains service result records
'''
# create service accessor and open file
self.serviceaccessor.open(filename)
status[self.servicename]['total'] = self.serviceaccessor.count()
status[self.servicename]['processed'] = 0
# loop through all results and store in database
while True:
filerecord = self.serviceaccessor.next()
if not filerecord: break
# transform to result attributes
result = Record()
result.source = self.servicename
# app.logger.debug('filerecord = {}'.format(filerecord.__dict__))
self.service2norm.transform(filerecord, result)
# app.logger.debug('result = {}'.format(result.__dict__))
# maybe we have a record in the database which matches this one, if so update the record
# otherwise create a new database record
## first get runner
runner = Runner.query.filter_by(club_id=club_id, name=result.runnername, dateofbirth=result.dob, gender=result.gender).first()
if not runner:
raise ParameterError, "could not find runner in database: {} line {} {} {} {}".format(filename, status[self.servicename]['processed']+2, result.runnername, result.dob, result.gender)
## next get race
### Race has uniqueconstraint for club_id/name/year/fixeddist. It's been seen that there are additional races in athlinks,
### but just assume the first is the correct one.
raceyear = ftime.asc2dt(result.date).year
race = Race.query.filter_by(club_id=club_id, name=result.racename, year=raceyear, fixeddist=race_fixeddist(result.distmiles)).first()
# races = Race.query.filter_by(club_id=club_id, name=result.racename, date=result.date, fixeddist=race_fixeddist(result.distmiles)).all()
# race = None
# for thisrace in races:
# if abs(thisrace.distance - result.distmiles) < RACEEPSILON:
# race = thisrace
# break
if not race:
raise ParameterError, "could not find race in database: {} line {} {} {} {}".format(filename, status[self.servicename]['processed']+2, result.racename, result.date, result.distmiles)
## update or create result in database
try:
agage = age(ftime.asc2dt(race.date), ftime.asc2dt(runner.dateofbirth))
result.agpercent, result.agtime, result.agfactor = ag.agegrade(agage, runner.gender, result.distmiles, result.timesecs)
dbresult = RaceResult(club_id, runner.id, race.id, None, result.timesecs, runner.gender, agage, instandings=False)
for attr in ['agfactor', 'agtime', 'agpercent', 'source', 'sourceid', 'sourceresultid', 'fuzzyage']:
setattr(dbresult,attr,getattr(result,attr))
insert_or_update(db.session, RaceResult, dbresult, skipcolumns=['id'],
club_id=club_id, source=self.servicename, runnerid=runner.id, raceid=race.id)
# maybe user is trying to cancel
except SystemExit:
raise
# otherwise just log and ignore result
except:
app.logger.warning('exception for "{}", result ignored, processing {} result {}\n{}'.format(runner.name, self.servicename, result.__dict__, traceback.format_exc()))
# update the number of results processed and pass back the status
status[self.servicename]['lastname'] = result.runnername
status[self.servicename]['processed'] += 1
thistask.update_state(state='PROGRESS', meta={'progress':status})
# finished reading results, close input file
self.serviceaccessor.close()
loclist = []
for f in ['Street 1', 'City','State']:
if r[f]:
loclist.append( str(r[f]) )
loc = ', '.join(loclist)
if r['Zip']:
loc += ' ' + str(r['Zip'])
# since this is init function there is no location record yet
locdata = gmaps.get_location(loc, None, app.config['GMAPS_CACHE_LIMIT'])
locrec = Location.query.filter_by(id=locdata['id']).one()
return locrec
locsmapping['location'] = maploc
locsmapping['icon'] = lambda r: Icon.query.filter_by(icon=r['Icon']).one() if r['Icon'] else None
locsmapping['iconsubtype'] = lambda r: IconSubtype.query.filter_by(iconsubtype=r['Business Type']).one() if r['Business Type'] else None
locsx = Transform(locsmapping, sourceattr=False, targetattr=True)
locshdr = next(locsrows) # pulls header
locscols = len(locsxhdr)
for in_loc in locsrows:
# seems to be more rows than those which have data, so break out at first empty row
if in_loc[0] == None: break
thisloc = dict(zip(locshdr[:locscols], in_loc[:locscols]))
out_loc = IconLocation()
app.logger.info('processing {}'.format(thisloc))
locsx.transform(thisloc, out_loc)
def post(self):
try:
val = DteFormValidate(ApplnValidator(allow_extra_fields=True))
results = val.validate(request.form)
if results['results']:
raise ParameterError(results['results'])
formdata = results['python']
name = formdata['name']
config = RacingTeamConfig.query.filter_by(
**localinterest_query_params()).one_or_none()
interest = Interest.query.filter_by(interest=g.interest).one()
if not config:
raise ParameterError(
'interest configuration needs to be created')
# we're logging this now
logtime = datetime.now()
applnrec = RacingTeamApplication(interest=localinterest(),
logtime=logtime)
applnformfields = 'name,email,dob,gender,applntype,comments'.split(
',')
applndbfields = 'name,email,dateofbirth,gender,type,comments'.split(
',')
applnmapping = dict(zip(applndbfields, applnformfields))
form2appln = Transform(applnmapping, sourceattr=False)
form2appln.transform(request.form, applnrec)
db.session.add(applnrec)
# race result information
mailfields = OrderedDict([
('name', 'Name'),
('email', 'Email'),
('dob', 'Birth Date'),
('gender', 'Gender'),
('race1_name', 'Race 1 - Name'),
('race1_location', 'Race 1 - Location'),
('race1_date', 'Race 1 - Date'),
('race1_age', 'Race 1 - Age'),
('race1_distance', 'Race 1 - Distance'),
('race1_units', ''),
('race1_time', 'Race 1 - Official Time (hh:mm:ss)'),
('race1_resultslink', 'Race 1 - Results Website'),
('race1_agegrade', 'Race 1 - Age Grade'),
('race2_name', 'Race 2 - Name'),
('race2_location', 'Race 2 - Location'),
('race2_date', 'Race 2 - Date'),
('race2_age', 'Race 2 - Age'),
('race2_distance', 'Race 2 - Distance'),
('race2_units', ''),
('race2_time', 'Race 2 - Official Time (hh:mm:ss)'),
('race2_resultslink', 'Race 2 - Results Website'),
('race2_agegrade', 'Race 2 - Age Grade'),
('comments', 'Comments'),
])
for ndx in [1, 2]:
resultsrec = RacingTeamResult(interest=localinterest())
resultsform = f'race{ndx}_name,race{ndx}_date,race{ndx}_location,race{ndx}_resultslink,race{ndx}_distance,race{ndx}_units,race{ndx}_time,race{ndx}_agegrade,race{ndx}_age'.split(
',')
resultsdb = 'eventname,eventdate,location,url,distance,units,time,agegrade,age'.split(
',')
resultsmapping = dict(zip(resultsdb, resultsform))
resultsxform = Transform(resultsmapping, sourceattr=False)
resultsxform.transform(request.form, resultsrec)
resultsrec.application = applnrec
db.session.add(resultsrec)
# commit database changes
db.session.commit()
# send confirmation email
subject = f"[racing-team-application] New racing team application from {name}"
body = div()
with body:
p('The following application for the racing team was submitted. If this is correct, '
f'no action is required. If you have any changes, please contact {config.fromemail}'
)
with table(), tbody():
for field in mailfields:
with tr():
td(mailfields[field])
td(request.form[field])
with p():
text(f'Racing Team - {config.fromemail}')
br()
text(f'{interest.description}')
html = body.render()
tolist = formdata['email']
fromlist = config.fromemail
cclist = config.applnccemail
sendmail(subject, fromlist, tolist, html, ccaddr=cclist)
return jsonify({'status': 'OK'})
except Exception as e:
db.session.rollback()
exc_type, exc_value, exc_traceback = exc_info()
current_app.logger.error(''.join(format_tb(exc_traceback)))
error = format_exc()
return jsonify({'status': 'error', 'error': escape(repr(e))})
def importmembers(configfile, debug=False, stats=False):
#----------------------------------------------------------------------
'''
import member data to mailchimp
configfile must have the following
[mailchimp]
RSU_CLUB: <runsignup club_id>
RSU_KEY: <key from runsignup partnership>
RSU_SECRET: <secret from runsignup partnership>
MC_KEY: <api key from MailChimp>
MC_LIST: <name of list of interest>
MC_GROUPNAMES: groupname1,groupname2,...
MC_SHADOWCATEGORY: <name of shadowcategory>
* shadowcategory groups are used to show desired inclusion but the group itself under other categories can be toggled by subscriber
* this is used to prevent the recipient from being added back to the group against their wishes
* if a groupname is not also under shadowcategory, it is only ticked if the subscriber was not present in the list prior to import
* this category's group names include all of the group names which are reserved for members
MC_CURRMEMBERGROUP: <name of group which is set for current members>
MC_PASTMEMBERGROUP: <name of group which is set for current and past members>
:param configfile: name of configuration file
:param debug: set to True for debug output
:param stats: set to True for stats output (INFO)
'''
# set up logging
thislogger.propagate = True
if debug:
# set up debug logging
thislogger.setLevel(logging.DEBUG)
elif stats:
# INFO logging
thislogger.setLevel(logging.INFO)
else:
# WARNING logging
thislogger.setLevel(logging.WARNING)
# load configuration
rsuconfig = getitems(configfile, 'runsignup')
mcconfig = getitems(configfile, 'mailchimp')
club_id = rsuconfig['RSU_CLUB']
rsukey = rsuconfig['RSU_KEY']
rsusecret = rsuconfig['RSU_SECRET']
mckey = mcconfig['MC_KEY']
mclist = mcconfig['MC_LIST']
mcgroupnames = mcconfig['MC_GROUPNAMES'].split(',')
mcshadowcategory = mcconfig['MC_SHADOWCATEGORY']
mcpastmembergroupname = mcconfig['MC_PASTMEMBERGROUP']
mccurrmembergroupname = mcconfig['MC_CURRMEMBERGROUP']
mctimeout = float(mcconfig['MC_TIMEOUT'])
# use Transform to simplify RunSignUp format
xform = Transform(
{
'last':
lambda mem: mem['user']['last_name'],
'first':
lambda mem: mem['user']['first_name'],
'email':
lambda mem: mem['user']['email'] if 'email' in mem['user'] else '',
'primary':
lambda mem: mem['primary_member'] == 'T',
'start':
'membership_start',
'end':
'membership_end',
'modified':
'last_modified',
},
# source and target are dicts, not objects
sourceattr=False,
targetattr=False)
# download current member list from RunSignUp
# get current members from RunSignUp, transforming each to local format
# only save one member per email address, primary member preferred
rsu = RunSignUp(key=rsukey, secret=rsusecret, debug=debug)
rsu.open()
rsumembers = rsu.members(club_id)
rsucurrmembers = {}
for rsumember in rsumembers:
memberrec = {}
xform.transform(rsumember, memberrec)
memberkey = memberrec['email'].lower()
# only save if there's an email address
# the primary member takes precedence, but if different email for nonprimary members save those as well
if memberkey and (memberrec['primary']
or memberkey not in rsucurrmembers):
rsucurrmembers[memberkey] = memberrec
rsu.close()
# It's important not to add someone back to a group which they've decided not to receive
# emails from. For this reason, a membergroup is defined with the same group names as
# the real groups the user is interested in, for those groups which don't make up the whole
# list.
# download categories / groups from MailChimp
client = MailChimp(mc_api=mckey, timeout=mctimeout)
lists = client.lists.all(get_all=True, fields="lists.name,lists.id")
list_id = [lst['id'] for lst in lists['lists'] if lst['name'] == mclist][0]
categories = client.lists.interest_categories.all(
list_id=list_id, fields="categories.title,categories.id")
# groups are for anyone, shadowgroups are for members only
groups = {}
shadowgroups = {}
# for debugging
allgroups = {}
for category in categories['categories']:
mcgroups = client.lists.interest_categories.interests.all(
list_id=list_id,
category_id=category['id'],
fields="interests.name,interests.id")
for group in mcgroups['interests']:
# save for debug
allgroups[group['id']] = '{} / {}'.format(category['title'],
group['name'])
# special group to track past members
if group['name'] == mcpastmembergroupname:
mcpastmembergroup = group['id']
# and current members
elif group['name'] == mccurrmembergroupname:
mccurrmembergroup = group['id']
# newly found members are enrolled in all groups
elif category['title'] != mcshadowcategory:
groups[group['name']] = group['id']
# shadowgroups is used to remember the state of member's only groups for previous members
# if a member's membership has expired they must be removed from any group(s) which have the same name as
# those within the shadowgroup(s) (associated groups)
# additionally if any nonmembers are found, they must be removed from the associated groups
# this last bit can happen if someone who is not a member tries to enroll in a members only group
else:
shadowgroups[group['name']] = group['id']
# set up specific groups for mc api
mcapi = Obj()
# members new to the list get all the groups
mcapi.newmember = {
id: True
for id in list(groups.values()) + list(shadowgroups.values()) +
[mccurrmembergroup] + [mcpastmembergroup]
}
# previous members who lapsed get the member groups disabled
mcapi.nonmember = {
id: False
for id in [
groups[gname]
for gname in list(groups.keys()) if gname in shadowgroups
] + [mccurrmembergroup]
}
# members groups set to True, for mcapi.unsubscribed merge
mcapi.member = {
id: True
for id in list(shadowgroups.values()) + [
groups[gname]
for gname in list(groups.keys()) if gname in shadowgroups
] + [mccurrmembergroup] + [mcpastmembergroup]
}
# unsubscribed members who previously were not past members get member groups turned on and 'other' groups turned off
mcapi.unsubscribed = merge_dicts(
mcapi.member, {
id: False
for id in [
groups[gname]
for gname in list(groups.keys()) if gname not in shadowgroups
]
})
# retrieve all members of this mailchimp list
# key these into dict by id (md5 has of lower case email address)
tmpmcmembers = client.lists.members.all(
list_id=list_id,
get_all=True,
fields=
'members.id,members.email_address,members.status,members.merge_fields,members.interests'
)
mcmembers = {}
for mcmember in tmpmcmembers['members']:
mcmembers[mcmember['id']] = mcmember
# collect some stats
stat = Stat([
'addedtolist', 'newmemberunsubscribed', 'newmember', 'pastmember',
'nonmember', 'memberunsubscribedskipped', 'membercleanedskipped',
'mailchimperror'
])
# loop through club members
# if club member is in mailchimp
# make sure shadowgroups are set (but don't change groups as these may have been adjusted by club member)
# don't change subscribed status
# pop off mcmembers as we want to deal with the leftovers later
# if club member is not already in mailchimp
# add assuming all groups (groups + shadowgroups)
for memberkey in rsucurrmembers:
clubmember = rsucurrmembers[memberkey]
mcmemberid = mcid(clubmember['email'])
thislogger.debug('processing {} {}'.format(clubmember['email'],
mcmemberid))
# if club member is in mailchimp
if mcmemberid in mcmembers:
mcmember = mcmembers.pop(mcmemberid)
# check if any changes are required
# change required if current member not set
if not mcmember['interests'][mccurrmembergroup]:
# if not past member, just set the needful
if not mcmember['interests'][mcpastmembergroup]:
# if subscribed, all groups are set
if mcmember['status'] == 'subscribed':
client.lists.members.update(
list_id=list_id,
subscriber_hash=mcmemberid,
data={'interests': mcapi.newmember})
stat.newmember += 1
# if unsubscribed, subscribe them to member stuff, but remove everything else
elif mcmember['status'] == 'unsubscribed':
try:
client.lists.members.update(
list_id=list_id,
subscriber_hash=mcmemberid,
data={
'interests': mcapi.unsubscribed,
'status': 'subscribed'
})
stat.newmemberunsubscribed += 1
# MailChimp won't let us resubscribe this member
except MailChimpError as e:
thislogger.info(
'member unsubscribed, skipped: {}'.format(
clubmember['email']))
stat.memberunsubscribedskipped += 1
# other statuses are skipped
else:
thislogger.info('member cleaned, skipped: {}'.format(
clubmember['email']))
stat.membercleanedskipped += 1
# past member, recall what they had set before for the member stuff
else:
pastmemberinterests = merge_dicts(
{
groups[gname]:
mcmember['interests'][shadowgroups[gname]]
for gname in list(shadowgroups.keys())
}, {mccurrmembergroup: True})
client.lists.members.update(
list_id=list_id,
subscriber_hash=mcmemberid,
data={'interests': pastmemberinterests})
stat.pastmember += 1
# if club member is missing from mailchimp
else:
try:
client.lists.members.create(list_id=list_id,
data={
'email_address':
clubmember['email'],
'merge_fields': {
'FNAME':
clubmember['first'],
'LNAME': clubmember['last']
},
'interests':
mcapi.newmember,
'status':
'subscribed'
})
stat.addedtolist += 1
except MailChimpError as e:
ed = e.args[0]
thislogger.warning('MailChimpError {} for {}: {}'.format(
ed['title'], clubmember['email'], ed['detail']))
stat.mailchimperror += 1
# at this point, mcmembers have only those enrollees who are not in the club
# loop through each of these and make sure club only interests are removed
for mcmemberid in mcmembers:
mcmember = mcmembers[mcmemberid]
# change required if current member set
if mcmember['interests'][mccurrmembergroup]:
# save member interests for later if they rejoin
memberinterests = {
shadowgroups[gname]: mcmember['interests'][groups[gname]]
for gname in shadowgroups
}
client.lists.members.update(list_id=list_id,
subscriber_hash=mcmemberid,
data={
'interests':
merge_dicts(
mcapi.nonmember,
memberinterests)
})
stat.nonmember += 1
# log stats
thislogger.info(stat)
def get(self):
#----------------------------------------------------------------------
try:
# verify user can read the data, otherwise abort
if not self.readpermission():
db.session.rollback()
flask.abort(403)
# DataTables options string, data: and buttons: are passed separately
dt_options = {
'dom': '<"H"lBpfr>t<"F"i>',
'columns': [],
'ordering': True,
'serverSide': False,
}
dt_options.update(self.dtoptions)
# set up columns
if hasattr(self.columns, '__call__'):
columns = self.columns()
else:
columns = self.columns
for column in columns:
dt_options['columns'].append(column)
# set up buttons
if hasattr(self.buttons, '__call__'):
buttons = self.buttons()
else:
buttons = self.buttons
# set up column transformation from header items to data items
mapping = { c['data']:c['label'] for c in columns }
headers2data = Transform(mapping, sourceattr=False, targetattr=False)
# build table data
if hasattr(self.csvfile, '__call__'):
csvfile = self.csvfile()
else:
csvfile = self.csvfile
with open(csvfile, 'rb') as _CSV:
tabledata = []
CSV = DictReader(_CSV)
for csvrow in CSV:
datarow = {}
headers2data.transform(csvrow, datarow)
tabledata.append(datarow)
# commit database updates and close transaction
db.session.commit()
# render page
return flask.render_template('datatables.html',
pagename = self.pagename,
pagejsfiles = addscripts(['datatables.js', 'buttons.colvis.js']),
tabledata = tabledata,
tablebuttons = buttons,
options = {'dtopts': dt_options},
inhibityear = True, # NOTE: prevents common DatatablesCsv
)
except:
# roll back database updates and close transaction
db.session.rollback()
raise
rt_appln_dbmapping = dict(zip(rt_appln_dbattrs, rt_appln_formfields))
rt_appln_formmapping = dict(zip(rt_appln_formfields, rt_appln_dbattrs))
rt_appln_formmapping['logtime'] = lambda dbrow: timestamp.dt2asc(dbrow.logtime)
rt_appln_formmapping['dateofbirth'] = lambda dbrow: isodate.dt2asc(dbrow.
dateofbirth)
result_dbattrs = 'eventdate,eventname,location,url,distance,units,time,age,agegrade'.split(
',')
db2client = {}
for ndx in [1, 2]:
result_clientfields = f'race{ndx}_eventdate,race{ndx}_eventname,race{ndx}_location,race{ndx}_resultslink,race{ndx}_distance,race{ndx}_units,' \
f'race{ndx}_time,race{ndx}_age,race{ndx}_agegrade'.split(',')
clientmapping = dict(zip(result_clientfields, result_dbattrs))
clientmapping[f'race{ndx}_eventdate'] = lambda dbrow: isodate.dt2asc(
dbrow.eventdate) if dbrow.eventdate else None
db2client[ndx] = Transform(clientmapping, targetattr=False)
class RacingTeamApplnsView(DbCrudApiInterestsRolePermissions):
def _result2client(self, result, raceid):
client = {}
db2client[raceid].transform(result, client)
return client
def nexttablerow(self):
dbrec = next(self.rows)
clientrec = self.dte.get_response_data(dbrec)
results = SortedKeyList(dbrec.rt_results, key=lambda a: a.eventdate)
results = list(results)
# may be one or more empty results
while len(results) < 2:
def update(interest, membershipfile):
"""update member, membership tables, from membershipfile if supplied, or from service based on interest"""
thislogger = getLogger('members.cli')
if debug:
thislogger.setLevel(DEBUG)
else:
thislogger.setLevel(INFO)
thislogger.propagate = True
# set local interest
g.interest = interest
linterest = localinterest()
# assume update will complete ok
tableupdatetime = TableUpdateTime.query.filter_by(
interest=linterest, tablename='member').one_or_none()
if not tableupdatetime:
tableupdatetime = TableUpdateTime(interest=linterest,
tablename='member')
db.session.add(tableupdatetime)
tableupdatetime.lastchecked = datetime.today()
# normal case is download from RunSignUp service
if not membershipfile:
# get, check club id
club_id = linterest.service_id
if not (linterest.club_service == 'runsignup' and club_id):
raise ParameterError(
'interest Club Service must be runsignup, and Service ID must be defined'
)
# transform: membership "file" format from RunSignUp API
xform = Transform(
{
'MemberID':
lambda mem: mem['user']['user_id'],
'MembershipID':
'membership_id',
'MembershipType':
'club_membership_level_name',
'FamilyName':
lambda mem: mem['user']['last_name'],
'GivenName':
lambda mem: mem['user']['first_name'],
'MiddleName':
lambda mem: mem['user']['middle_name']
if mem['user']['middle_name'] else '',
'Gender':
lambda mem: 'Female'
if mem['user']['gender'] == 'F' else 'Male',
'DOB':
lambda mem: mem['user']['dob'],
'City':
lambda mem: mem['user']['address']['city'],
'State':
lambda mem: mem['user']['address']['state'],
'Email':
lambda mem: mem['user']['email']
if 'email' in mem['user'] else '',
'PrimaryMember':
'primary_member',
'JoinDate':
'membership_start',
'ExpirationDate':
'membership_end',
'LastModified':
'last_modified',
},
sourceattr=False, # source and target are dicts
targetattr=False)
rsu = RunSignUp(key=current_app.config['RSU_KEY'],
secret=current_app.config['RSU_SECRET'],
debug=debug)
def doxform(ms):
membership = {}
xform.transform(ms, membership)
return membership
with rsu:
# get current and future members from RunSignUp, and put into common format
rawmemberships = rsu.members(club_id, current_members_only='F')
currfuturememberships = [
m for m in rawmemberships
if m['membership_end'] >= datetime.today().date().isoformat()
]
memberships = [doxform(ms) for ms in currfuturememberships]
# membershipfile supplied
else:
with open(membershipfile, 'r') as _MF:
MF = DictReader(_MF)
# memberships already in common format
memberships = [ms for ms in MF]
# sort memberships by member (family_name, given_name, gender, dob), expiration_date
memberships.sort(key=lambda m: (m['FamilyName'], m['GivenName'], m[
'Gender'], m['DOB'], m['ExpirationDate']))
# set up member, membership transforms to create db records
# transform: member record from membership "file" format
memxform = Transform(
{
'family_name':
'FamilyName',
'given_name':
'GivenName',
'middle_name':
'MiddleName',
'gender':
'Gender',
'svc_member_id':
'MemberID',
'dob':
lambda m: isodate.asc2dt(m['DOB']).date(),
'hometown':
lambda m: f'{m["City"]}, {m["State"]}'
if 'City' in m and 'State' in m else '',
'email':
'Email',
'start_date':
lambda m: isodate.asc2dt(m['JoinDate']).date(),
'end_date':
lambda m: isodate.asc2dt(m['ExpirationDate']).date(),
},
sourceattr=False,
targetattr=True)
# transform: update member record from membership record
memupdate = Transform(
{
'svc_member_id': 'svc_member_id',
'hometown': 'hometown',
'email': 'email',
},
sourceattr=True,
targetattr=True)
# transform: membership record from membership "file" format
mshipxform = Transform(
{
'svc_member_id':
'MemberID',
'svc_membership_id':
'MembershipID',
'membershiptype':
'MembershipType',
'hometown':
lambda m: f'{m["City"]}, {m["State"]}'
if 'City' in m and 'State' in m else '',
'email':
'Email',
'start_date':
lambda m: isodate.asc2dt(m['JoinDate']).date(),
'end_date':
lambda m: isodate.asc2dt(m['ExpirationDate']).date(),
'primary':
lambda m: m['PrimaryMember'].lower() == 't' or m['PrimaryMember'].
lower() == 'yes',
'last_modified':
lambda m: rsudt.asc2dt(m['LastModified']),
},
sourceattr=False,
targetattr=True)
# insert member, membership records
for m in memberships:
# need MembershipId to be string for comparison with database key
m['MembershipID'] = str(m['MembershipID'])
filternamedob = and_(Member.family_name == m['FamilyName'],
Member.given_name == m['GivenName'],
Member.gender == m['Gender'],
Member.dob == isodate.asc2dt(m['DOB']))
# func.binary forces case sensitive comparison. see https://stackoverflow.com/a/31788828/799921
filtermemberid = Member.svc_member_id == func.binary(m['MemberID'])
filtermember = or_(filternamedob, filtermemberid)
# get all the member records for this member
# note there may currently be more than one member record, as the memberships may be discontiguous
thesemembers = SortedList(key=lambda member: member.end_date)
thesemembers.update(Member.query.filter(filtermember).all())
# if member doesn't exist, create member and membership records
if len(thesemembers) == 0:
thismember = Member(interest=localinterest())
memxform.transform(m, thismember)
db.session.add(thismember)
# flush so thismember can be referenced in thismship, and can be found in later processing
db.session.flush()
thesemembers.add(thismember)
thismship = Membership(interest=localinterest(), member=thismember)
mshipxform.transform(m, thismship)
db.session.add(thismship)
# flush so thismship can be found in later processing
db.session.flush()
# if there are already some memberships for this member, merge with this membership (m)
else:
# dbmships is keyed by svc_membership_id, sorted by end_date
# NOTE: membership id is unique only within a member -- be careful if the use of dbmships changes
# to include multiple members
dbmships = ItemSortedDict(lambda k, v: v.end_date)
for thismember in thesemembers:
for mship in thismember.memberships:
dbmships[mship.svc_membership_id] = mship
# add membership if not already there for this member
mshipid = m['MembershipID']
if mshipid not in dbmships:
newmship = True
thismship = Membership(interest=localinterest())
db.session.add(thismship)
# flush so thismship can be found in later processing
db.session.flush()
# update existing membership
else:
newmship = False
thismship = dbmships[mshipid]
# merge the new membership record into the database record
mshipxform.transform(m, thismship)
# add new membership to data structure
if newmship:
dbmships[thismship.svc_membership_id] = thismship
# need list view for some processing
dbmships_keys = dbmships.keys()
# check for overlaps
for thisndx in range(1, len(dbmships_keys)):
prevmship = dbmships[dbmships_keys[thisndx - 1]]
thismship = dbmships[dbmships_keys[thisndx]]
if thismship.start_date <= prevmship.end_date:
oldstart = thismship.start_date
newstart = prevmship.end_date + timedelta(1)
oldstartasc = isodate.dt2asc(oldstart)
newstartasc = isodate.dt2asc(newstart)
endasc = isodate.dt2asc(thismship.end_date)
memberkey = f'{m["FamilyName"]},{m["GivenName"]},{m["DOB"]}'
thislogger.warn(
f'overlap detected for {memberkey}: end={endasc} was start={oldstartasc} now start={newstartasc}'
)
thismship.start_date = newstart
# update appropriate member record(s), favoring earlier member records
# NOTE: membership hometown, email get copied into appropriate member records;
# since mship list is sorted, last one remains
for mshipid in dbmships_keys:
mship = dbmships[mshipid]
for nextmndx in range(len(thesemembers)):
thismember = thesemembers[nextmndx]
lastmember = thesemembers[nextmndx -
1] if nextmndx != 0 else None
# TODO: use Transform for these next four entries
# corner case: someone changed their birthdate
thismember.dob = isodate.asc2dt(m['DOB']).date()
# prefer last name found
thismember.given_name = m['GivenName']
thismember.family_name = m['FamilyName']
thismember.middle_name = m['MiddleName'] if m[
'MiddleName'] else ''
# mship causes new member record before this one
# or after end of thesemembers
# or wholy between thesemembers
if (mship.end_date + timedelta(1) < thismember.start_date
or (nextmndx == len(thesemembers) - 1)
and mship.start_date >
thismember.end_date + timedelta(1)
or lastmember and mship.start_date >
lastmember.end_date + timedelta(1)
and mship.end_date < thismember.start_date):
newmember = Member(interest=localinterest())
# flush so thismember can be referenced in mship, and can be found in later processing
db.session.flush()
memxform.transform(m, newmember)
mship.member = newmember
break
# mship extends this member record from the beginning
if mship.end_date + timedelta(1) == thismember.start_date:
thismember.start_date = mship.start_date
mship.member = thismember
memupdate.transform(mship, thismember)
break
# mship extends this member from the end
if mship.start_date == thismember.end_date + timedelta(1):
thismember.end_date = mship.end_date
mship.member = thismember
memupdate.transform(mship, thismember)
break
# mship end date was changed
if (mship.start_date >= thismember.start_date
and mship.start_date <= thismember.end_date
and mship.end_date != thismember.end_date):
thismember.end_date = mship.end_date
mship.member = thismember
memupdate.transform(mship, thismember)
break
# mship start date was changed
if (mship.end_date >= thismember.start_date
and mship.end_date <= thismember.end_date
and mship.start_date != thismember.start_date):
thismember.start_date = mship.start_date
mship.member = thismember
memupdate.transform(mship, thismember)
break
# mship wholly contained within this member
if mship.start_date >= thismember.start_date and mship.end_date <= thismember.end_date:
mship.member = thismember
memupdate.transform(mship, thismember)
break
# delete unused member records
delmembers = []
for mndx in range(len(thesemembers)):
thismember = thesemembers[mndx]
if len(thismember.memberships) == 0:
delmembers.append(thismember)
for delmember in delmembers:
db.session.delete(delmember)
thesemembers.remove(delmember)
if len(delmembers) > 0:
db.session.flush()
# merge member records as appropriate
thisndx = 0
delmembers = []
for nextmndx in range(1, len(thesemembers)):
thismember = thesemembers[thisndx]
nextmember = thesemembers[nextmndx]
if thismember.end_date + timedelta(1) == nextmember.start_date:
for mship in nextmember.memberships:
mship.member = thismember
delmembers.append(nextmember)
else:
thisndx = nextmndx
for delmember in delmembers:
db.session.delete(delmember)
if len(delmembers) > 0:
db.session.flush()
# save statistics file
groupfolder = join(current_app.config['APP_FILE_FOLDER'], interest)
if not exists(groupfolder):
mkdir(groupfolder, mode=0o770)
statspath = join(groupfolder, current_app.config['APP_STATS_FILENAME'])
analyzemembership(statsfile=statspath)
# make sure we remember everything we did
db.session.commit()