def _getdivision(member):
#----------------------------------------------------------------------
'''
gets division as of Jan 1 from RunningAheadMember record
:param member: RunningAheadMember record
:rtype: division text
'''
# use local time
today = time.time()-time.timezone
todaydt = timeu.epoch2dt(today)
jan1 = datetime(todaydt.year, 1, 1)
memberage = timeu.age(jan1, ymd.asc2dt(member.dob))
# this must match grand prix configuration in membership database
# TODO: add api to query this information from scoretility
if memberage <= 13:
div = '13 and under'
elif memberage <= 29:
div = '14-29'
elif memberage <= 39:
div = '30-39'
elif memberage <= 49:
div = '40-49'
elif memberage <= 59:
div = '50-59'
elif memberage <= 69:
div = '60-69'
else:
div = '70 and over'
return div
def login():
#----------------------------------------------------------------------
# define form
form = LoginForm()
# Validate form input
#if flask.request.method == "POST" and form.validate_on_submit():
if form.validate_on_submit():
try:
# Retrieve the user from the datastore
user = racedb.find_user(form.email.data)
# flag user doesn't exist or incorrect password
if not (user and user.check_password(form.password.data)):
return flask.render_template('login.html', form=form, error='username or password invalid')
# we're good
# Keep the user info in the session using Flask-Login
login_user(user)
flask.session['logged_in'] = True
flask.session['user_name'] = user.name
flask.session.permanent = True
# Tell Flask-Principal the identity changed
identity_changed.send(
flask.current_app._get_current_object(),
identity = Identity(user.id))
userclubs = getuserclubs(user)
# zero clubs is an internal error in the databse
if not(userclubs):
db.session.rollback()
raise dbConsistencyError,'no clubs found in database'
# give user access to the first club in the list if no club already chosen
# club_choices set in nav module. If this club.id is not in club_choices,
# need to reset to first available
if 'club_id' not in flask.session or flask.session['club_id'] not in [c[0] for c in userclubs]:
club = Club.query.filter_by(id=userclubs[0][0]).first()
flask.session['club_id'] = club.id
flask.session['club_name'] = club.name
# set default year to be current year
today = timeu.epoch2dt(time.time())
flask.session['year'] = today.year
# log login
app.logger.debug("logged in user '{}'".format(flask.session['user_name']))
# commit database updates and close transaction
db.session.commit()
return flask.redirect(flask.request.args.get('next') or flask.url_for('index'))
except:
# roll back database updates and close transaction
db.session.rollback()
raise
return flask.render_template('login.html', form=form)
def _getdivision(member):
'''
gets division as of Jan 1
:param member: Member record
:rtype: division text
'''
# use local time
today = time.time() - time.timezone
todaydt = epoch2dt(today)
jan1 = datetime(todaydt.year, 1, 1)
memberage = age(jan1, member.dob)
# this must match grand prix configuration in membership database
# TODO: add api to query this information from scoretility
if memberage <= 13:
div = '13 and under'
elif memberage <= 29:
div = '14-29'
elif memberage <= 39:
div = '30-39'
elif memberage <= 49:
div = '40-49'
elif memberage <= 59:
div = '50-59'
elif memberage <= 69:
div = '60-69'
else:
div = '70 and over'
return div
def getresults(self, name, fname, lname, gender, dt_dob, begindate, enddate):
#----------------------------------------------------------------------
'''
retrieves a list of results for a single name
must be overridden when ResultsCollect is instantiated
use dt_dob to filter errant race results, based on age of runner on race day
:param name: name of participant for which results are to be returned
:param fname: first name of participant
:param lname: last name of participant
:param gender: 'M' or 'F'
:param dt_dob: participant's date of birth, as datetime
:param begindate: epoch time for start of results, 00:00:00 on date to begin
:param end: epoch time for end of results, 23:59:59 on date to finish
:rtype: list of serviceresults, each of which can be processed by convertresult
'''
# remember participant data
self.name = name
self.fname = fname
self.lname = lname
self.gender = gender
self.dt_dob = dt_dob
self.dob = ftime.dt2asc(dt_dob)
# get results for this athlete
allresults = self.service.listresults(fname,lname)
# filter by date and by age
filteredresults = []
for result in allresults:
e_racedate = ftime.asc2epoch(result.racedate)
# skip result if outside the desired time window
if e_racedate < begindate or e_racedate > enddate: continue
# skip result if runner's age doesn't match the age within the result
dt_racedate = timeu.epoch2dt(e_racedate)
racedateage = timeu.age(dt_racedate,dt_dob)
if result.age != racedateage: continue
# skip result if runner's gender doesn't match gender within the result
resultgen = result.gender
if resultgen != gender: continue
# if we reach here, the result is ok, and is added to filteredresults
filteredresults.append(result)
# back to caller
return filteredresults
def getlocation(self, address):
#----------------------------------------------------------------------
'''
retrieve location from database, if available, else get from googlemaps api
:param address: address for lookup
:rtype: Location instance
'''
dbaddress = address
if len(dbaddress) > MAX_LOCATION_LEN:
dbaddress = dbaddress[0:MAX_LOCATION_LEN]
loc = Location.query.filter_by(name=dbaddress).first()
now = epoch2dt(time.time())
if not loc or (now - loc.cached_at > CACHE_REFRESH):
# new location
loc = Location(name=dbaddress)
# get geocode from google
# use the full address, not dbaddress which gets s
gc = geocode(self.client, address=address)
# if we got good data, fill in the particulars
# assume first in list is good, give warning if multiple entries received back
if gc:
# notify if multiple values returned
if len(gc) > 1:
app.logger.warning('geocode: multiple locations ({}) received from googlemaps for {}'.format(len(gc), address))
# save lat/long from first value returned
loc.latitude = gc[0]['geometry']['location']['lat']
loc.longitude = gc[0]['geometry']['location']['lng']
# if no response, still store in database, but flag as error
else:
loc.lookuperror = True
# remember when last retrieved
loc.cached_at = now
# insert or update -- flush is done within, so id should be set after this
insert_or_update(db.session, Location, loc, skipcolumns=['id'], name=dbaddress)
# and back to caller
return loc
def main():
#----------------------------------------------------------------------
'''
update club membership information
'''
parser = argparse.ArgumentParser(version='{0} {1}'.format('runningclub',version.__version__))
parser.add_argument('memberfile',help='csv, xls or xlsx file with member information')
parser.add_argument('-r','--racedb',help='filename of race database (default is as configured during rcuserconfig)',default=None)
parser.add_argument('--debug',help='if set, create updatemembers.txt for debugging',action='store_true')
args = parser.parse_args()
OUT = None
if args.debug:
OUT = open('updatemembers.txt','w')
racedb.setracedb(args.racedb)
session = racedb.Session()
# get clubmembers from file
memberfile = args.memberfile
root,ext = os.path.splitext(memberfile)
if ext in ['.xls','.xlsx']:
members = clubmember.XlClubMember(memberfile)
elif ext in ['.csv']:
members = clubmember.CsvClubMember(memberfile)
else:
print '***ERROR: invalid memberfile {}, must be csv, xls or xlsx'.format(memberfile)
return
# get old clubmembers from database
dbmembers = clubmember.DbClubMember() # use default database
# get all the member runners currently in the database
# hash them into dict by (name,dateofbirth)
allrunners = session.query(racedb.Runner).filter_by(member=True,active=True).all()
inactiverunners = {}
for thisrunner in allrunners:
inactiverunners[thisrunner.name,thisrunner.dateofbirth] = thisrunner
if OUT:
OUT.write('found id={0}, runner={1}\n'.format(thisrunner.id,thisrunner))
# make report for new members found with this memberfile
logdir = os.path.dirname(args.memberfile)
memberfilebase = os.path.splitext(os.path.basename(args.memberfile))[0]
newmemlogname = '{0}-newmem.csv'.format(memberfilebase)
NEWMEM = open(os.path.join(logdir,newmemlogname),'wb')
NEWMEMCSV = csv.DictWriter(NEWMEM,['name','dob'])
NEWMEMCSV.writeheader()
# prepare for age check
thisyear = timeu.epoch2dt(time.time()).year
asofasc = '{}-1-1'.format(thisyear) # jan 1 of current year
asof = tYmd.asc2dt(asofasc)
# process each name in new membership list
allmembers = members.getmembers()
for name in allmembers:
thesemembers = allmembers[name]
# NOTE: may be multiple members with same name
for thismember in thesemembers:
thisname = thismember['name']
thisdob = thismember['dob']
thisgender = thismember['gender'][0].upper() # male -> M, female -> F
thishometown = thismember['hometown']
# prep for if .. elif below by running some queries
# handle close matches, if DOB does match
age = timeu.age(asof,tYmd.asc2dt(thisdob))
matchingmember = dbmembers.findmember(thisname,age,asofasc)
dbmember = None
if matchingmember:
membername,memberdob = matchingmember
if memberdob == thisdob:
dbmember = racedb.getunique(session,racedb.Runner,member=True,name=membername,dateofbirth=thisdob)
# TODO: need to handle case where dob transitions from '' to actual date of birth
# no member found, maybe there is nonmember of same name already in database
if dbmember is None:
dbnonmember = racedb.getunique(session,racedb.Runner,member=False,name=thisname)
# TODO: there's a slim possibility that there are two nonmembers with the same name, but I'm sure we've already
# bolloxed that up in importresult as there's no way to discriminate between the two
# make report for new members
NEWMEMCSV.writerow({'name':thisname,'dob':thisdob})
# see if this runner is a member in the database already, or was a member once and make the update
# add or update runner in database
# get instance, if it exists, and make any updates
found = False
if dbmember is not None:
thisrunner = racedb.Runner(membername,thisdob,thisgender,thishometown)
# this is also done down below, but must be done here in case member's name has changed
if (thisrunner.name,thisrunner.dateofbirth) in inactiverunners:
inactiverunners.pop((thisrunner.name,thisrunner.dateofbirth))
# overwrite member's name if necessary
thisrunner.name = thisname
added = racedb.update(session,racedb.Runner,dbmember,thisrunner,skipcolumns=['id'])
found = True
# if runner's name is in database, but not a member, see if this runner is a nonmemember which can be converted
# Check first result for age against age within the input file
# if ages match, convert nonmember to member
elif dbnonmember is not None:
# get dt for date of birth, if specified
try:
dob = tYmd.asc2dt(thisdob)
except ValueError:
dob = None
# nonmember came into the database due to a nonmember race result, so we can use any race result to check nonmember's age
if dob:
result = session.query(racedb.RaceResult).filter_by(runnerid=dbnonmember.id).first()
resultage = result.agage
racedate = tYmd.asc2dt(result.race.date)
expectedage = racedate.year - dob.year - int((racedate.month, racedate.day) < (dob.month, dob.day))
# we found the right person, always if dob isn't specified, but preferably check race result for correct age
if dob is None or resultage == expectedage:
thisrunner = racedb.Runner(thisname,thisdob,thisgender,thishometown)
added = racedb.update(session,racedb.Runner,dbnonmember,thisrunner,skipcolumns=['id'])
found = True
else:
print '{} found in database, wrong age, expected {} found {} in {}'.format(thisname,expectedage,resultage,result)
# TODO: need to make file for these, also need way to force update, because maybe bad date in database for result
# currently this will cause a new runner entry
# if runner was not found in database, just insert new runner
if not found:
thisrunner = racedb.Runner(thisname,thisdob,thisgender,thishometown)
added = racedb.insert_or_update(session,racedb.Runner,thisrunner,skipcolumns=['id'],name=thisname,dateofbirth=thisdob)
# remove this runner from collection of runners which should be deactivated in database
if (thisrunner.name,thisrunner.dateofbirth) in inactiverunners:
inactiverunners.pop((thisrunner.name,thisrunner.dateofbirth))
if OUT:
if added:
OUT.write('added or updated {0}\n'.format(thisrunner))
else:
OUT.write('no updates necessary {0}\n'.format(thisrunner))
# any runners remaining in 'inactiverunners' should be deactivated
for (name,dateofbirth) in inactiverunners:
thisrunner = session.query(racedb.Runner).filter_by(name=name,dateofbirth=dateofbirth).first() # should be only one returned by filter
thisrunner.active = False
if OUT:
OUT.write('deactivated {0}\n'.format(thisrunner))
session.commit()
session.close()
NEWMEM.close()
if OUT:
OUT.close()
def collect(searchfile,outfile,begindate,enddate):
#----------------------------------------------------------------------
'''
collect race results from runningahead
:param searchfile: path to file containing names, genders, birth dates to search for
:param outfile: output file path
:param begindate: epoch time - choose races between begindate and enddate
:param enddate: epoch time - choose races between begindate and enddate
'''
outfilehdr = 'GivenName,FamilyName,name,DOB,Gender,race,date,age,miles,km,time'.split(',')
# open files
_IN = open(searchfile,'rb')
IN = csv.DictReader(_IN)
_OUT = open(outfile,'wb')
OUT = csv.DictWriter(_OUT,outfilehdr)
OUT.writeheader()
# common fields between input and output
commonfields = 'GivenName,FamilyName,DOB,Gender'.split(',')
# create runningahead access, grab users who have used the steeplechasers.org portal to RA
ra = runningahead.RunningAhead()
users = ra.listusers()
rausers = []
for user in users:
rauser = ra.getuser(user['token'])
rausers.append((user,rauser))
# reset begindate to beginning of day, enddate to end of day
dt_begindate = timeu.epoch2dt(begindate)
a_begindate = fdate.dt2asc(dt_begindate)
adj_begindate = datetime.datetime(dt_begindate.year,dt_begindate.month,dt_begindate.day,0,0,0)
e_begindate = timeu.dt2epoch(adj_begindate)
dt_enddate = timeu.epoch2dt(enddate)
a_enddate = fdate.dt2asc(dt_enddate)
adj_enddate = datetime.datetime(dt_enddate.year,dt_enddate.month,dt_enddate.day,23,59,59)
e_enddate = timeu.dt2epoch(adj_enddate)
# get today's date for high level age filter
start = time.time()
today = timeu.epoch2dt(start)
# loop through runners in the input file
for runner in IN:
fname,lname = runner['GivenName'],runner['FamilyName']
membername = '{} {}'.format(fname,lname)
log.debug('looking for {}'.format(membername))
e_dob = fdate.asc2epoch(runner['DOB'])
dt_dob = fdate.asc2dt(runner['DOB'])
dob = runner['DOB']
gender = runner['Gender'][0]
# find thisuser
foundmember = False
for user,rauser in rausers:
if 'givenName' not in rauser or 'birthDate' not in rauser: continue # we need to know the name and birth date
givenName = rauser['givenName'] if 'givenName' in rauser else ''
familyName = rauser['familyName'] if 'familyName' in rauser else ''
rausername = '******'.format(givenName,familyName)
if rausername == membername and dt_dob == fdate.asc2dt(rauser['birthDate']):
foundmember = True
log.debug('found {}'.format(membername))
break
# member is not this ra user, keep looking
# if we couldn't find this member in RA, try the next member
if not foundmember: continue
## skip getting results if participant too young
#todayage = timeu.age(today,dt_dob)
#if todayage < 14: continue
# if we're here, found the right user, now let's look at the workouts
workouts = ra.listworkouts(user['token'],begindate=a_begindate,enddate=a_enddate,getfields=FIELD['workout'].keys())
# save race workouts, if any found
results = []
if workouts:
for wo in workouts:
if wo['workoutName'].lower() != 'race': continue
if 'duration' not in wo['details']: continue # seen once, not sure why
thisdate = wo['date']
dt_thisdate = fdate.asc2dt(thisdate)
thisdist = runningahead.dist2meters(wo['details']['distance'])
thistime = wo['details']['duration']
thisrace = wo['course']['name'] if wo.has_key('course') else 'unknown'
if thistime == 0:
log.warning('{} has 0 time for {} {}'.format(membername,thisrace,thisdate))
continue
stat = {'GivenName':fname,'FamilyName':lname,'name':membername,
'DOB':dob,'Gender':gender,'race':thisrace,'date':thisdate,'age':timeu.age(dt_thisdate,dt_dob),
'miles':thisdist/METERSPERMILE,'km':thisdist/1000.0,'time':render.rendertime(thistime,0)}
results.append(stat)
# loop through each result
for result in results:
e_racedate = fdate.asc2epoch(result['date'])
# skip result if outside the desired time window
if e_racedate < begindate or e_racedate > enddate: continue
# create output record and copy fields
outrec = result
resulttime = result['time']
# strange case of TicksString = ':00'
if resulttime[0] == ':':
resulttime = '0'+resulttime
while resulttime.count(':') < 2:
resulttime = '0:'+resulttime
outrec['time'] = resulttime
OUT.writerow(outrec)
_OUT.close()
_IN.close()
finish = time.time()
print 'elapsed time (min) = {}'.format((finish-start)/60)
def collect(searchfile,outfile,begindate,enddate):
#----------------------------------------------------------------------
'''
collect race results from athlinks
:param searchfile: path to file containing names, genders, birth dates to search for
:param outfile: output file path
:param begindate: epoch time - choose races between begindate and enddate
:param enddate: epoch time - choose races between begindate and enddate
'''
# open files
_IN = open(searchfile,'rb')
IN = csv.DictReader(_IN)
_OUT = open(outfile,'wb')
OUT = csv.DictWriter(_OUT,resultfilehdr)
OUT.writeheader()
# common fields between input and output
commonfields = 'GivenName,FamilyName,DOB,Gender'.split(',')
# create athlinks
athl = athlinks.Athlinks(debug=True)
# reset begindate to beginning of day, enddate to end of day
dt_begindate = timeu.epoch2dt(begindate)
adj_begindate = datetime.datetime(dt_begindate.year,dt_begindate.month,dt_begindate.day,0,0,0)
begindate = timeu.dt2epoch(adj_begindate)
dt_enddate = timeu.epoch2dt(enddate)
adj_enddate = datetime.datetime(dt_enddate.year,dt_enddate.month,dt_enddate.day,23,59,59)
enddate = timeu.dt2epoch(adj_enddate)
# get today's date for high level age filter
start = time.time()
today = timeu.epoch2dt(start)
# loop through runners in the input file
for runner in IN:
name = ' '.join([runner['GivenName'],runner['FamilyName']])
e_dob = ftime.asc2epoch(runner['DOB'])
dt_dob = ftime.asc2dt(runner['DOB'])
## skip getting results if participant too young
#todayage = timeu.age(today,dt_dob)
#if todayage < 14: continue
# get results for this athlete
results = athl.listathleteresults(name)
# loop through each result
for result in results:
e_racedate = athlinks.gettime(result['Race']['RaceDate'])
# skip result if outside the desired time window
if e_racedate < begindate or e_racedate > enddate: continue
# create output record and copy common fields
outrec = {}
for field in commonfields:
outrec[field] = runner[field]
# skip result if runner's age doesn't match the age within the result
# sometimes athlinks stores the age group of the runner, not exact age,
# so also check if this runner's age is within the age group, and indicate if so
dt_racedate = timeu.epoch2dt(e_racedate)
racedateage = timeu.age(dt_racedate,dt_dob)
resultage = int(result['Age'])
if resultage != racedateage:
# if results are not stored as age group, skip this result
if (resultage/5)*5 != resultage:
continue
# result's age might be age group, not exact age
else:
# if runner's age consistent with race age, use result, but mark "fuzzy"
if (racedateage/5)*5 == resultage:
outrec['fuzzyage'] = 'Y'
# otherwise skip result
else:
continue
# skip result if runner's gender doesn't match gender within the result
resultgen = result['Gender'][0]
if resultgen != runner['Gender'][0]: continue
# get course used for this result
course = athl.getcourse(result['Race']['RaceID'],result['CourseID'])
# skip result if not Running or Trail Running race
thiscategory = course['Courses'][0]['RaceCatID']
if thiscategory not in race_category: continue
# fill in output record fields from runner, result, course
# combine name, get age
outrec['name'] = '{} {}'.format(runner['GivenName'],runner['FamilyName'])
outrec['age'] = result['Age']
# leave athlmember and athlid blank if result not from an athlink member
athlmember = result['IsMember']
if athlmember:
outrec['athlmember'] = 'Y'
outrec['athlid'] = result['RacerID']
# race name, location; convert from unicode if necessary
# TODO: make function to do unicode translation -- apply to runner name as well (or should csv just store unicode?)
racename = csvu.unicode2ascii(course['RaceName'])
coursename = csvu.unicode2ascii(course['Courses'][0]['CourseName'])
outrec['race'] = '{} / {}'.format(racename,coursename)
outrec['date'] = ftime.epoch2asc(athlinks.gettime(course['RaceDate']))
outrec['loc'] = csvu.unicode2ascii(course['Home'])
# distance, category, time
distmiles = athlinks.dist2miles(course['Courses'][0]['DistUnit'],course['Courses'][0]['DistTypeID'])
distkm = athlinks.dist2km(course['Courses'][0]['DistUnit'],course['Courses'][0]['DistTypeID'])
if distkm < 0.050: continue # skip timed events, which seem to be recorded with 0 distance
outrec['miles'] = distmiles
outrec['km'] = distkm
outrec['category'] = race_category[thiscategory]
resulttime = result['TicksString']
# strange case of TicksString = ':00'
if resulttime[0] == ':':
resulttime = '0'+resulttime
while resulttime.count(':') < 2:
resulttime = '0:'+resulttime
outrec['time'] = resulttime
# just leave out age grade if exception occurs
try:
agpercent,agresult,agfactor = ag.agegrade(racedateage,resultgen,distmiles,timeu.timesecs(resulttime))
outrec['ag'] = agpercent
if agpercent < 15 or agpercent >= 100: continue # skip obvious outliers
except:
pass
OUT.writerow(outrec)
_OUT.close()
_IN.close()
finish = time.time()
print 'number of URLs retrieved = {}'.format(athl.geturlcount())
print 'elapsed time (min) = {}'.format((finish-start)/60)
def post(self):
#----------------------------------------------------------------------
def allowed_file(filename):
return '.' in filename and filename.split('.')[-1] in ['csv','xlsx','xls']
try:
club_id = flask.session['club_id']
thisyear = flask.session['year']
readcheck = ViewClubDataPermission(club_id)
writecheck = UpdateClubDataPermission(club_id)
# verify user can write the data, otherwise abort
if not writecheck.can():
db.session.rollback()
flask.abort(403)
# if using api, collect data from api and save in temp directory
useapi = request.args.get('useapi')=='true'
# if we're using the api, do some quick checks that the request makes sense
# save apitype, apiid, apikey, apisecret for later
if useapi:
thisclub = Club.query.filter_by(id=club_id).first()
apitype = thisclub.memberserviceapi
apiid = thisclub.memberserviceid
if not apitype or not apiid:
db.session.rollback()
cause = 'Unexpected Error: API requested but not configured'
app.logger.error(cause)
return failure_response(cause=cause)
thisapi = ApiCredentials.query.filter_by(name=apitype).first()
if not thisapi:
db.session.rollback()
cause = "Unexpected Error: API credentials for '{}' not configured".format(apitype)
app.logger.error(cause)
return failure_response(cause=cause)
apikey = thisapi.key
apisecret = thisapi.secret
if not apikey or not apisecret:
db.session.rollback()
cause = "Unexpected Error: API credentials for '{}' not configured with key or secret".format(apitype)
app.logger.error(cause)
return failure_response(cause=cause)
# if we're not using api, file came in with request
else:
memberfile = request.files['file']
# get file extention
root,ext = os.path.splitext(memberfile.filename)
# make sure valid file
if not memberfile:
db.session.rollback()
cause = 'Unexpected Error: Missing file'
app.logger.error(cause)
return failure_response(cause=cause)
if not allowed_file(memberfile.filename):
db.session.rollback()
cause = 'Invalid file type {} for file {}'.format(ext,memberfile.filename)
app.logger.error(cause)
return failure_response(cause=cause)
# get all the member runners currently in the database
# hash them into dict by (name,dateofbirth)
allrunners = Runner.query.filter_by(club_id=club_id,member=True,active=True).all()
inactiverunners = {}
for thisrunner in allrunners:
inactiverunners[thisrunner.name,thisrunner.dateofbirth] = thisrunner
# if some members exist, verify user wants to overwrite
if allrunners and not request.args.get('force')=='true':
db.session.rollback()
return failure_response(cause='Overwrite members?',confirm=True)
# if we're using the api, collect the member information using the appropriate credentials
# NOTE: only runsignup supported at this time
if useapi:
tempdir = tempfile.mkdtemp()
memberfilename = 'members.csv'
ext = '.csv'
memberpathname = os.path.join(tempdir,memberfilename)
rsu_members2csv(apiid, apikey, apisecret, rsu_api2filemapping, filepath=memberpathname)
else:
# save file for import
tempdir = tempfile.mkdtemp()
memberfilename = secure_filename(memberfile.filename)
memberpathname = os.path.join(tempdir,memberfilename)
memberfile.save(memberpathname)
# bring in data from the file
if ext in ['.xls','.xlsx']:
members = clubmember.XlClubMember(memberpathname)
elif ext in ['.csv']:
members = clubmember.CsvClubMember(memberpathname)
# how did this happen? check allowed_file() for bugs
else:
db.session.rollback()
cause = 'Program Error: Invalid file type {} for file {} path {} (unexpected)'.format(ext,memberfilename,memberpathname)
app.logger.error(cause)
return failure_response(cause=cause)
# remove file and temporary directory
os.remove(memberpathname)
try:
os.rmdir(tempdir)
# no idea why this can happen; hopefully doesn't happen on linux
except WindowsError,e:
app.logger.debug('WindowsError exception ignored: {}'.format(e))
# get old clubmembers from database
dbmembers = clubmember.DbClubMember(club_id=club_id) # use default database
# prepare for age check
thisyear = timeu.epoch2dt(time.time()).year
asofasc = '{}-1-1'.format(thisyear) # jan 1 of current year
asof = tYmd.asc2dt(asofasc)
# process each name in new membership list
allmembers = members.getmembers()
for name in allmembers:
thesemembers = allmembers[name]
# NOTE: may be multiple members with same name
for thismember in thesemembers:
thisname = thismember['name']
thisfname = thismember['fname']
thislname = thismember['lname']
thisdob = thismember['dob']
thisgender = thismember['gender'][0].upper() # male -> M, female -> F
thishometown = thismember['hometown']
thisrenewdate = thismember['renewdate']
thisexpdate = thismember['expdate']
# prep for if .. elif below by running some queries
# handle close matches, if DOB does match
age = timeu.age(asof,tYmd.asc2dt(thisdob))
matchingmember = dbmembers.findmember(thisname,age,asofasc)
dbmember = None
if matchingmember:
membername,memberdob = matchingmember
if memberdob == thisdob:
dbmember = racedb.getunique(db.session,Runner,club_id=club_id,member=True,name=membername,dateofbirth=thisdob)
# TODO: need to handle case where dob transitions from '' to actual date of birth
# no member found, maybe there is nonmember of same name already in database
if dbmember is None:
dbnonmember = racedb.getunique(db.session,Runner,club_id=club_id,member=False,name=thisname)
# TODO: there's a slim possibility that there are two nonmembers with the same name, but I'm sure we've already
# bolloxed that up in importresult as there's no way to discriminate between the two
## make report for new members
#NEWMEMCSV.writerow({'name':thisname,'dob':thisdob})
# see if this runner is a member in the database already, or was a member once and make the update
# add or update runner in database
# get instance, if it exists, and make any updates
found = False
if dbmember is not None:
thisrunner = Runner(club_id,membername,thisdob,thisgender,thishometown,
fname=thisfname,lname=thislname,
renewdate=thisrenewdate,expdate=thisexpdate)
# this is also done down below, but must be done here in case member's name has changed
if (thisrunner.name,thisrunner.dateofbirth) in inactiverunners:
inactiverunners.pop((thisrunner.name,thisrunner.dateofbirth))
# overwrite member's name if necessary
thisrunner.name = thisname
added = racedb.update(db.session,Runner,dbmember,thisrunner,skipcolumns=['id'])
found = True
# if runner's name is in database, but not a member, see if this runner is a nonmemember which can be converted
# Check first result for age against age within the input file
# if ages match, convert nonmember to member
elif dbnonmember is not None:
# get dt for date of birth, if specified
try:
dob = tYmd.asc2dt(thisdob)
except ValueError:
dob = None
# nonmember came into the database due to a nonmember race result, so we can use any race result to check nonmember's age
if dob:
result = RaceResult.query.filter_by(runnerid=dbnonmember.id).first()
resultage = result.agage
racedate = tYmd.asc2dt(result.race.date)
expectedage = timeu.age(racedate,dob)
#expectedage = racedate.year - dob.year - int((racedate.month, racedate.day) < (dob.month, dob.day))
# we found the right person, always if dob isn't specified, but preferably check race result for correct age
if dob is None or resultage == expectedage:
thisrunner = Runner(club_id,thisname,thisdob,thisgender,thishometown,
fname=thisfname,lname=thislname,
renewdate=thisrenewdate,expdate=thisexpdate)
added = racedb.update(db.session,Runner,dbnonmember,thisrunner,skipcolumns=['id'])
found = True
else:
app.logger.warning('{} found in database, wrong age, expected {} found {} in {}'.format(thisname,expectedage,resultage,result))
# TODO: need to make file for these, also need way to force update, because maybe bad date in database for result
# currently this will cause a new runner entry
# if runner was not found in database, just insert new runner
if not found:
thisrunner = Runner(club_id,thisname,thisdob,thisgender,thishometown,
fname=thisfname,lname=thislname,
renewdate=thisrenewdate,expdate=thisexpdate)
added = racedb.insert_or_update(db.session,Runner,thisrunner,skipcolumns=['id'],club_id=club_id,name=thisname,dateofbirth=thisdob)
# remove this runner from collection of runners which should be deactivated in database
if (thisrunner.name,thisrunner.dateofbirth) in inactiverunners:
inactiverunners.pop((thisrunner.name,thisrunner.dateofbirth))
# any runners remaining in 'inactiverunners' should be deactivated
for (name,dateofbirth) in inactiverunners:
thisrunner = Runner.query.filter_by(club_id=club_id,name=name,dateofbirth=dateofbirth).first() # should be only one returned by filter
thisrunner.active = False
# commit database updates and close transaction
db.session.commit()
return success_response()
def render(aag,outfile,summaryfile,detailfile,minagegrade,minraces,mintrend,begindate,enddate):
#----------------------------------------------------------------------
'''
render collected results
:param outfile: output file name template, like '{who}-ag-analysis-{date}-{time}.png'
:param summaryfile: summary file name template (.csv), may include {date} field
:param detailfile: summary file name template (.csv), may include {date} field
:param minagegrade: minimum age grade
:param minraces: minimum races in the same year as enddate
:param mintrend: minimum races over the full period for trendline
:param begindate: render races between begindate and enddate, datetime
:param enddate: render races between begindate and enddate, datetime
'''
firstyear = begindate.year
lastyear = enddate.year
yearrange = range(firstyear,lastyear+1)
summfields = ['name','age','gender']
distcategories = ['overall'] + [TRENDLIMITS[tlimit][0] for tlimit in TRENDLIMITS]
for stattype in ['1yr agegrade','avg agegrade','trend','numraces','stderr','r-squared','pvalue']:
for distcategory in distcategories:
summfields.append('{}\n{}'.format(stattype,distcategory))
if stattype == 'numraces':
for year in yearrange:
summfields.append('{}\n{}'.format(stattype,year))
tfile = timeu.asctime('%Y-%m-%d')
summaryfname = summaryfile.format(date=tfile.epoch2asc(time.time()))
_SUMM = open(summaryfname,'wb')
SUMM = csv.DictWriter(_SUMM,summfields)
SUMM.writeheader()
detailfname = detailfile.format(date=tfile.epoch2asc(time.time()))
detlfields = ['name','dob','gender'] + analyzeagegrade.AgeGradeStat.attrs + ['distmiles','distkm','rendertime']
detlfields.remove('priority') # priority is internal
_DETL = open(detailfname,'wb')
DETL = csv.DictWriter(_DETL,detlfields,extrasaction='ignore')
DETL.writeheader()
# create a figure used for everyone -- required to save memory
fig = plt.figure()
# loop through each member we've recorded information about
for thisname in aag:
rendername = thisname.title()
# remove duplicate entries
aag[thisname].deduplicate()
# crunch the numbers, and remove entries less than minagegrade
aag[thisname].crunch() # calculate age grade for each result
stats = aag[thisname].get_stats()
#for stat in stats:
# if stat.ag < minagegrade:
# aag[thisname].del_stat(stat)
# write detailed file before filtering
name,gender,dob = aag[thisname].get_runner()
detlout = {'name':rendername,'gender':gender,'dob':tfile.dt2asc(dob)}
for stat in stats:
for attr in analyzeagegrade.AgeGradeStat.attrs:
detlout[attr] = getattr(stat,attr)
if attr == 'date':
detlout[attr] = tfile.dt2asc(detlout[attr])
# interpret some of the data from the raw stat
detlout['distkm'] = detlout['dist'] / 1000.0
detlout['distmiles'] = detlout['dist']/METERSPERMILE
rendertime = ren.rendertime(detlout['time'],0)
while len(rendertime.split(':')) < 3:
rendertime = '0:'+rendertime
detlout['rendertime'] = rendertime
DETL.writerow(detlout)
jan1 = tfile.asc2dt('{}-1-1'.format(lastyear))
runnerage = timeu.age(jan1,dob)
# filter out runners younger than 14
if runnerage < 14: continue
# filter out runners who have not run enough races
stats = aag[thisname].get_stats()
if enddate:
lastyear = enddate.year
else:
lastyear = timeu.epoch2dt(time.time()).year
lastyearstats = [s for s in stats if s.date.year==lastyear]
if len(lastyearstats) < minraces: continue
# set up output file name template
if outfile:
aag[thisname].set_renderfname(outfile)
# set up rendering parameters
aag[thisname].set_xlim(begindate,enddate)
aag[thisname].set_ylim(minagegrade,100)
aag[thisname].set_colormap([200,100*METERSPERMILE])
# clear figure, set up axes
fig.clear()
ax = fig.add_subplot(111)
# render the results
aag[thisname].render_stats(fig) # plot statistics
# set up to collect averages
avg = collections.OrderedDict()
# draw trendlines, write output
allstats = aag[thisname].get_stats()
avg['overall'] = mean([s.ag for s in allstats])
trend = aag[thisname].render_trendline(fig,'overall',color='k')
# retrieve output filename for hyperlink
# must be called after set_runner and set_renderfname
thisoutfile = aag[thisname].get_outfilename()
summout = {}
summout['name'] = '=HYPERLINK("{}","{}")'.format(thisoutfile,rendername)
summout['age'] = runnerage
summout['gender'] = gender
oneyrstats = [s.ag for s in allstats if s.date.year == lastyear]
if len(oneyrstats) > 0:
summout['1yr agegrade\noverall'] = mean(oneyrstats)
summout['avg agegrade\noverall'] = avg['overall']
if len(allstats) >= mintrend:
summout['trend\noverall'] = trend.slope
summout['stderr\noverall'] = trend.stderr
summout['r-squared\noverall'] = trend.rvalue**2
summout['pvalue\noverall'] = trend.pvalue
summout['numraces\noverall'] = len(allstats)
for year in yearrange:
summout['numraces\n{}'.format(year)] = len([s for s in allstats if s.date.year==year])
for tlimit in TRENDLIMITS:
distcategory,distcolor = TRENDLIMITS[tlimit]
tstats = [s for s in allstats if s.dist >= tlimit[0] and s.dist <= tlimit[1]]
if len(tstats) < mintrend: continue
avg[distcategory] = mean([s.ag for s in tstats])
trend = aag[thisname].render_trendline(fig,distcategory,thesestats=tstats,color=distcolor)
oneyrcategory = [s.ag for s in tstats if s.date.year == lastyear]
if len(oneyrcategory) > 0:
summout['1yr agegrade\n{}'.format(distcategory)] = mean(oneyrcategory)
summout['avg agegrade\n{}'.format(distcategory)] = avg[distcategory]
summout['trend\n{}'.format(distcategory)] = trend.slope
summout['stderr\n{}'.format(distcategory)] = trend.stderr
summout['r-squared\n{}'.format(distcategory)] = trend.rvalue**2
summout['pvalue\n{}'.format(distcategory)] = trend.pvalue
summout['numraces\n{}'.format(distcategory)] = len(tstats)
SUMM.writerow(summout)
# annotate with averages
avgstr = 'averages\n'
for lab in avg:
thisavg = int(round(avg[lab]))
avgstr += ' {}: {}%\n'.format(lab,thisavg)
avgstr += 'age (1/1/{}): {}'.format(lastyear,runnerage)
# TODO: add get_*lim() to aag -- xlim and ylim are currently side-effect of aag.render_stats()
x1,xn = ax.get_xlim()
y1,yn = ax.get_ylim()
xy = (x1+10,y1+10)
aag[thisname].render_annotate(fig,avgstr,xy)
# save file
aag[thisname].save(fig)
_SUMM.close()
_DETL.close()
def main():
#----------------------------------------------------------------------
parser = argparse.ArgumentParser(
version='{0} {1}'.format('running', version.__version__))
parser.add_argument('infile', help='file generated by racewx')
parser.add_argument('racename', help='race name')
args = parser.parse_args()
infile = args.infile
racename = args.racename
# get input
_WX = open(infile, 'rb')
WX = csv.DictReader(_WX)
wxdata = []
for wx in WX:
wxdata.append(wx)
_WX.close()
# for now, filter out all but the max 'exectime' entries
lastexec = max([int(wx['exectime']) for wx in wxdata])
while int(wxdata[0]['exectime']) != lastexec:
wxdata.pop(0)
# pull out fields to plot
wxplot = {}
plotfields = [
'time', 'temperature', 'windchill', 'heatindex', 'dewpoint',
'windSpeed', 'windBearing', 'cloudCover', 'precipProbability',
'precipIntensity', 'cloudCover'
]
for f in plotfields:
wxplot[f] = [float(wx[f]) if wx[f] != '' else None for wx in wxdata]
# get range on 30 minute boundaries
starttime = int(wxplot['time'][0])
fintime = int(wxplot['time'][-1])
adjstart = (starttime / (30 * 60)) * (
30 * 60) # rounds to next lowest 30 minute boundary
adjfin = (
(fintime - 1 + 30 * 60) /
(30 * 60)) * (30 * 60) # rounds to next highest 30 minute boundary
startdt = timeu.epoch2dt(adjstart)
findt = timeu.epoch2dt(adjfin)
# time zone stuff, based on starting point
lat = float(wxdata[0]['lat'])
lon = float(wxdata[0]['lon'])
tzid = racewx.gettzid(lat, lon)
tz = pytz.timezone(tzid)
wxplot['localtime'] = [
timeu.utcdt2tzdt(timeu.epoch2dt(tm), tzid) for tm in wxplot['time']
]
# plot data
fig = plt.figure()
ttitle = timeu.asctime('%m/%d/%Y')
racedate = ttitle.epoch2asc(wxplot['time'][0])
fdate = ttitle.epoch2asc(lastexec)
fig.suptitle(
'forecast for {race} {date}\nforecast date {fdate}\nPowered by Forecast.io'
.format(race=racename, date=racedate, fdate=fdate),
size='small')
# set some formatting parameters
lw = 0.5 # line width
windcolor = 'b'
legendx = 1.35
# plot control
exists = {}
for f in ['windchill', 'heatindex']:
exists[f] = len([it for it in wxplot[f] if it is not None]) != 0
for f in ['precipIntensity']:
exists[f] = len([it for it in wxplot[f] if it > 0.0]) != 0
# plot temperatures
ax1 = fig.add_subplot(311)
ax1.plot(wxplot['localtime'],
wxplot['temperature'],
'k-',
label='temperature',
linewidth=lw)
if exists['windchill']:
ax1.plot(wxplot['localtime'],
wxplot['windchill'],
'b-',
label='wind chill',
linewidth=lw)
if exists['heatindex']:
ax1.plot(wxplot['localtime'],
wxplot['heatindex'],
'r-',
label='heat index',
linewidth=lw)
ax1.plot(wxplot['localtime'],
wxplot['dewpoint'],
'g-',
label='dew point',
linewidth=lw)
ax1.set_xlim(startdt, findt)
fig.subplots_adjust(top=0.88, right=0.75, bottom=0.15)
hfmt = dates.DateFormatter('%H:%M', tz=tz)
ax1.xaxis.set_major_formatter(hfmt)
ax1.xaxis.set_major_locator(dates.MinuteLocator(interval=30))
ax1.grid('on')
plt.setp(ax1.get_xticklabels(), visible=False)
plt.setp(ax1.get_yticklabels(), fontsize='small')
ax1.set_ylabel('degrees \nFahrenheit', fontsize='small')
#font = fm.FontProperties(fname='Humor-Sans.ttf')
font = fm.FontProperties()
xsmallfont = copy.deepcopy(font)
xsmallfont.set_size('x-small')
ax1.legend(prop=xsmallfont, loc='upper right', bbox_to_anchor=(legendx, 1))
# plot wind
ax2 = fig.add_subplot(312)
ax2.plot(wxplot['localtime'],
wxplot['windSpeed'],
label='wind speed',
linewidth=lw,
color=windcolor)
# note polar-> rectangular flips x,y from standard transformation because theta is from North instead of East
# not sure why need to invert U and V to get barb to point in right direction. Maybe vector comes from U,V and points to origin?
U = [
-1 * wxplot['windSpeed'][i] *
math.sin(math.radians(wxplot['windBearing'][i]))
for i in range(len(wxplot['windSpeed']))
]
V = [
-1 * wxplot['windSpeed'][i] *
math.cos(math.radians(wxplot['windBearing'][i]))
for i in range(len(wxplot['windSpeed']))
]
xdates = dates.date2num(
wxplot['localtime']) # barbs requires floats, not datetime
ax2.barbs(xdates,
wxplot['windSpeed'],
U,
V,
length=5,
barbcolor=windcolor,
flagcolor=windcolor,
linewidth=lw)
ax2.set_xlim(dates.date2num(startdt), dates.date2num(findt))
miny, maxy = ax2.get_ylim()
ax2.set_ylim(round(miny * 0.8), round(maxy * 1.2))
ax2.xaxis.set_major_formatter(hfmt)
ax2.xaxis.set_major_locator(dates.MinuteLocator(interval=30))
ax2.grid('on')
#plt.setp(ax2.get_xticklabels(), rotation='vertical', fontsize='small')
plt.setp(ax2.get_xticklabels(), visible=False)
plt.setp(ax2.get_yticklabels(), fontsize='small')
ax2.set_ylabel('miles per hour', fontsize='small')
ax2.legend(prop=xsmallfont, loc='upper right', bbox_to_anchor=(legendx, 1))
ax3 = fig.add_subplot(313)
precipprob = [100 * (prob or 0) for prob in wxplot['precipProbability']]
cloudcover = [100 * (cover or 0) for cover in wxplot['cloudCover']]
ax3.plot(wxplot['localtime'],
precipprob,
label='rain probability',
linewidth=lw,
color='b')
ax3.plot(wxplot['localtime'],
cloudcover,
label='cloud cover',
linewidth=lw,
color='g')
ax3.set_ylabel('percent', fontsize='small')
ax3.set_xlim(dates.date2num(startdt), dates.date2num(findt))
ax3.xaxis.set_major_formatter(hfmt)
ax3.xaxis.set_major_locator(dates.MinuteLocator(interval=30))
ax3.grid('on')
ax3.set_ylim(0, 100)
plt.setp(ax3.get_xticklabels(), rotation='vertical', fontsize='small')
plt.setp(ax3.get_yticklabels(), fontsize='small')
ax3.legend(prop=xsmallfont,
loc='upper right',
bbox_to_anchor=(legendx, 1.1))
if exists['precipIntensity']:
ax4 = ax3.twinx()
#ax4.plot(wxplot['localtime'],wxplot['precipIntensity'],label='intensity', linewidth=lw, color='r')
#ax4.set_yscale('log')
ax4.semilogy(wxplot['localtime'],
wxplot['precipIntensity'],
label='intensity',
nonposy='mask',
linewidth=lw,
color='r')
#ax4.set_ylabel('precipitation 0.002 very light sprinkling, 0.017 light precipitation, 0.1 precipitation, and 0.4 very heavy precipitation')
ax4.set_ylabel('intensity', fontsize='small')
ax4.set_ylim(0, 0.5)
plt.setp(ax4.get_yticklabels(), fontsize='small')
ax4.legend(prop=xsmallfont,
loc='upper right',
bbox_to_anchor=(legendx, 0.75))
tfile = timeu.asctime('%Y-%m-%d')
fdate = tfile.epoch2asc(lastexec)
racename = re.sub('\s', '', racename) # remove whitespace
outfile = 'race-weather-{race}-{fdate}.png'.format(race=racename,
fdate=fdate)
fig.savefig(outfile, format='png')
def collect(searchfile, outfile, begindate, enddate):
#----------------------------------------------------------------------
'''
collect race results from ultrasignup
:param searchfile: path to file containing names, genders, birth dates to search for
:param outfile: output file path
:param begindate: epoch time - choose races between begindate and enddate
:param enddate: epoch time - choose races between begindate and enddate
'''
# open files
_IN = open(searchfile, 'rb')
IN = csv.DictReader(_IN)
_OUT = open(outfile, 'wb')
OUT = csv.DictWriter(_OUT, UltraSignupResultFile.filehdr)
OUT.writeheader()
# common fields between input and output
commonfields = 'GivenName,FamilyName,DOB,Gender'.split(',')
# create ultrasignup access
ultra = ultrasignup.UltraSignup(debug=True)
# reset begindate to beginning of day, enddate to end of day
dt_begindate = timeu.epoch2dt(begindate)
adj_begindate = datetime.datetime(dt_begindate.year, dt_begindate.month,
dt_begindate.day, 0, 0, 0)
begindate = timeu.dt2epoch(adj_begindate)
dt_enddate = timeu.epoch2dt(enddate)
adj_enddate = datetime.datetime(dt_enddate.year, dt_enddate.month,
dt_enddate.day, 23, 59, 59)
enddate = timeu.dt2epoch(adj_enddate)
# get today's date for high level age filter
start = time.time()
today = timeu.epoch2dt(start)
# loop through runners in the input file
for runner in IN:
fname, lname = runner['GivenName'], runner['FamilyName']
e_dob = ftime.asc2epoch(runner['DOB'])
dt_dob = ftime.asc2dt(runner['DOB'])
gender = runner['Gender'][0]
## skip getting results if participant too young
#todayage = timeu.age(today,dt_dob)
#if todayage < 14: continue
# get results for this athlete
results = ultra.listresults(fname, lname)
# loop through each result
for result in results:
e_racedate = ftime.asc2epoch(result.racedate)
# skip result if outside the desired time window
if e_racedate < begindate or e_racedate > enddate: continue
# skip result if runner's age doesn't match the age within the result
dt_racedate = timeu.epoch2dt(e_racedate)
racedateage = timeu.age(dt_racedate, dt_dob)
if result.age != racedateage: continue
# skip result if runner's gender doesn't match gender within the result
resultgen = result.gender
if resultgen != runner['Gender'][0]: continue
# create output record and copy common fields
outrec = {}
for field in commonfields:
outrec[field] = runner[field]
# fill in output record fields from runner, result
# combine name, get age
outrec['name'] = '{} {}'.format(runner['GivenName'],
runner['FamilyName'])
outrec['age'] = result.age
# race name, location; convert from unicode if necessary
racename = result.racename
outrec['race'] = racename
outrec['date'] = ftime.epoch2asc(e_racedate)
outrec['loc'] = '{}, {}'.format(result.racecity, result.racestate)
# distance, category, time
distmiles = result.distmiles
distkm = result.distkm
if distkm is None or distkm < 0.050:
continue # should already be filtered within ultrasignup, but just in case
outrec['miles'] = distmiles
outrec['km'] = distkm
resulttime = result.racetime
# int resulttime means DNF, most likely -- skip this result
if type(resulttime) == int: continue
# strange case of TicksString = ':00'
if resulttime[0] == ':':
resulttime = '0' + resulttime
while resulttime.count(':') < 2:
resulttime = '0:' + resulttime
outrec['time'] = resulttime
# just leave out age grade if exception occurs
try:
agpercent, agresult, agfactor = ag.agegrade(
racedateage, gender, distmiles, timeu.timesecs(resulttime))
outrec['ag'] = agpercent
if agpercent < 15 or agpercent >= 100:
continue # skip obvious outliers
except:
pass
OUT.writerow(outrec)
_OUT.close()
_IN.close()
finish = time.time()
print 'number of URLs retrieved = {}'.format(ultra.geturlcount())
print 'elapsed time (min) = {}'.format((finish - start) / 60)
def analyzemembership(memberfileh, participantfileh, detailfile=None):
#----------------------------------------------------------------------
'''
analyze event registration proximity to member joindate
:param memberfileh: membership file handle, individual records
:param paraticipantfileh: event participant file handle
:param detailfile: (optional) name of file for detailed output
:rtype: OrderedDict {numdays: count, ...} - numdays is number of days event registration minus membership join date
'''
# debug
if detailfile:
_DETL = open(detailfile,'wb')
detailhdr = 'eventname,dob,membername,email,status,joindate,registered,join2event'.split(',')
DETL = csv.DictWriter(_DETL,detailhdr)
DETL.writeheader()
# pull in memberfile, participants file
membership = RunningAheadMembers(memberfileh)
event = RunningAheadParticipants(participantfileh)
# iterate through member participants
participants = event.activeregistrations_iter()
# what is expiration date for current members
now = timeu.epoch2dt(time.time())
currentmemberexp = ymd.dt2asc(datetime(now.year,12,31))
# loop through event registrants, keeping a histogram of how many days since member joined before registering for the event
# if member joined after registering for the event, the number will be negative
hist = {}
for participant in participants:
# gather event participant information
lname = participant.lname
fname = participant.fname
searchname = '{} {}'.format(fname,lname)
dob = participant.dob
asc_regdate = participant.registrationdate # like '1/28/2015 11:56:53 AM'
dt_regdate = mdy.asc2dt(asc_regdate.split(' ')[0])
asc_regdate = ymd.dt2asc(dt_regdate) # convert to yyyy-mm-dd
d_regdate = date(dt_regdate.year, dt_regdate.month, dt_regdate.day)
email = participant.email
# create record for detailfile
detailrec = {'eventname':searchname,'dob':dob,'registered':asc_regdate,'email':email}
# try to find in member file
key = membership.getmemberkey(lname,fname,dob)
if key:
detailrec['status'] = 'match'
else:
closemembers = membership.getclosematchkeys()
if len(closemembers) >= 1:
# take best match
key = closemembers[0]
detailrec['status'] = 'close'
else:
detailrec['status'] = 'missed'
# if found some member who appears to be the right person
if key:
# get name of found member
member = membership.getmember(key)
membername = '{} {}'.format(member.fname, member.lname)
detailrec['membername'] = membername
# figure out the join date
asc_joindate = member.join
dt_joindate = ymd.asc2dt(asc_joindate)
d_joindate = date(dt_joindate.year, dt_joindate.month, dt_joindate.day)
detailrec['joindate'] = asc_joindate
# check for lapsed members
if member.expiration != currentmemberexp:
detailrec['status'] = 'lapsed'
# how many days since member joined until member registered for event
join2event = (d_regdate - d_joindate).days
detailrec['join2event'] = join2event
# add one to this histogram count
hist[join2event] = hist.setdefault(join2event,0) + 1
# debug
if detailfile:
DETL.writerow(detailrec)
# debug
if detailfile:
_DETL.close()
# create orderered histogram
allcounts = hist.keys()
allcounts.sort()
ordhist = OrderedDict()
for y in range(min(allcounts),max(allcounts)+1):
ordhist[y] = hist[y] if y in hist else 0
return ordhist
def analyzemembership(memberfileh, participantfileh, detailfile=None):
#----------------------------------------------------------------------
'''
analyze event registration proximity to member joindate
:param memberfileh: membership file handle, individual records
:param paraticipantfileh: event participant file handle
:param detailfile: (optional) name of file for detailed output
:rtype: OrderedDict {numdays: count, ...} - numdays is number of days event registration minus membership join date
'''
# debug
if detailfile:
_DETL = open(detailfile, 'w', newline='')
detailhdr = 'eventname,dob,membername,email,status,joindate,registered,join2event'.split(
',')
DETL = csv.DictWriter(_DETL, detailhdr)
DETL.writeheader()
# pull in memberfile, participants file
membership = RunningAheadMembers(memberfileh)
event = RunningAheadParticipants(participantfileh)
# iterate through member participants
participants = event.activeregistrations_iter()
# what is expiration date for current members
now = timeu.epoch2dt(time.time())
currentmemberexp = ymd.dt2asc(datetime(now.year, 12, 31))
# loop through event registrants, keeping a histogram of how many days since member joined before registering for the event
# if member joined after registering for the event, the number will be negative
hist = {}
for participant in participants:
# gather event participant information
lname = participant.lname
fname = participant.fname
searchname = '{} {}'.format(fname, lname)
dob = participant.dob
asc_regdate = participant.registrationdate # like '1/28/2015 11:56:53 AM'
dt_regdate = mdy.asc2dt(asc_regdate.split(' ')[0])
asc_regdate = ymd.dt2asc(dt_regdate) # convert to yyyy-mm-dd
d_regdate = date(dt_regdate.year, dt_regdate.month, dt_regdate.day)
email = participant.email
# create record for detailfile
detailrec = {
'eventname': searchname,
'dob': dob,
'registered': asc_regdate,
'email': email
}
# try to find in member file
key = membership.getmemberkey(lname, fname, dob)
if key:
detailrec['status'] = 'match'
else:
closemembers = membership.getclosematchkeys()
if len(closemembers) >= 1:
# take best match
key = closemembers[0]
detailrec['status'] = 'close'
else:
detailrec['status'] = 'missed'
# if found some member who appears to be the right person
if key:
# get name of found member
member = membership.getmember(key)
membername = '{} {}'.format(member.fname, member.lname)
detailrec['membername'] = membername
# figure out the join date
asc_joindate = member.join
dt_joindate = ymd.asc2dt(asc_joindate)
d_joindate = date(dt_joindate.year, dt_joindate.month,
dt_joindate.day)
detailrec['joindate'] = asc_joindate
# check for lapsed members
if member.expiration != currentmemberexp:
detailrec['status'] = 'lapsed'
# how many days since member joined until member registered for event
join2event = (d_regdate - d_joindate).days
detailrec['join2event'] = join2event
# add one to this histogram count
hist[join2event] = hist.setdefault(join2event, 0) + 1
# debug
if detailfile:
DETL.writerow(detailrec)
# debug
if detailfile:
_DETL.close()
# create orderered histogram
allcounts = sorted(list(hist.keys()))
ordhist = OrderedDict()
for y in range(min(allcounts), max(allcounts) + 1):
ordhist[y] = hist[y] if y in hist else 0
return ordhist
def analyzemembership(memberfileh,detailfile=None,overlapfile=None):
#----------------------------------------------------------------------
'''
compare membership statistics, year on year
:param memberfileh: membership file handle, individual records
:param detailfile: optional detailed debug file name
:param overlapfile: optional overlap debug file name to record overlapping join / expiration date periods
:rtype: OrderedDict {year: OrderedDict {datetime:count,...}, ...}
'''
# debug
if detailfile:
_DETL = open(detailfile,'wb')
DETL = csv.DictWriter(_DETL,['ord','effective','name','catchup',
# 'renewal',
'join','expiration'])
DETL.writeheader()
detlrecord = 0
# pull in memberfile
members = RunningAheadMembers(memberfileh,overlapfile=overlapfile)
# iterate through memberships
memberships = members.membership_iter()
## loop through preprocessed records
years = {}
for membership in memberships:
# asc_renewaldate = membership.renew
asc_joindate = membership.join
asc_expdate = membership.expiration
# renewaldate = ymd.asc2dt(asc_renewaldate)
joindate = ymd.asc2dt(asc_joindate)
expdate = ymd.asc2dt(asc_expdate)
fname = membership.fname
lname = membership.lname
dob = membership.dob
fullname = '{}, {}'.format(lname,fname)
# when clicking "Export individual records", joindate is the effective date for the specific year
effectivedate = joindate
## increment the member count for the member's effective date
year = effectivedate.year
# good data starts in 2013
if year >= 2013:
# create year if it hasn't been created
if year not in years:
years[year] = {}
# increment the effectivedate date within the year
years[year][effectivedate] = years[year].get(effectivedate,0) + 1
# debug
if detailfile:
detlrecord += 1
DETL.writerow({'effective':ymd.dt2asc(effectivedate),'name':fullname,
# 'renewal':asc_renewaldate,
'join':asc_joindate,'expiration':asc_expdate,
'ord':detlrecord})
# for all years after effectivedate's until expdate's, increment jan 1
# this happens if there is a grace period and member's renewal counts for following year,
# and for multiyear membership
# NOTE: this is not under "if year >= 2013" because there are some joindates in 2012,
# captured here under jan1, 2013
for y in range(effectivedate.year+1,expdate.year+1):
jan1 = datetime(y,1,1)
if y not in years:
years[y] = {}
years[y][jan1] = years[y].get(jan1,0) + 1
# debug
if detailfile:
detlrecord += 1
DETL.writerow({'effective':ymd.dt2asc(jan1),'name':fullname,
# 'renewal':asc_renewaldate,
'join':asc_joindate,'expiration':asc_expdate,
'catchup':'y',
'ord':detlrecord})
# debug
if detailfile:
_DETL.close()
# create an entry with 0 count for today's date, if none exists (use local time)
# if, say, last entry was July 28, this will have the effect of prettying up the output
# for full month rendering
today = timeu.epoch2dt(time.time()-time.timezone)
thisyear = today.year
thismonth = today.month
thisday = today.day
today = datetime(thisyear, thismonth, thisday) # removes time of day from today
if today not in years[thisyear]:
years[thisyear][today] = 0
# remove any entries for any years accumulated after this year
# this can happen if long expiration dates are in database
for y in range(thisyear+1, max(years.keys())+1):
years.pop(y)
# create orderered dicts
allyears = years.keys()
allyears.sort()
ordyears = OrderedDict()
for y in allyears:
ordyears[y] = OrderedDict()
# make sure each month has entry in first and final date, so annotations in rendermembershipanalysis work nicely
thismonth = 1
for thisitem in sorted(years[y].items(), key=lambda t: t[0]):
thisdate = thisitem[0]
# if we are at a new month, create an empty entry for the first day of this month
# need while loop in case there was a whole month without memberships
# also make sure there is an entry the last date of the previous month
while (thismonth != thisdate.month):
thismonth += 1
# make sure there is entry the last date of the previous month
prevmonth = thismonth - 1
lastdayprevmonth = monthrange(y,prevmonth)[1]
lastdateprevmonth = datetime(y,thismonth-1,lastdayprevmonth)
if lastdateprevmonth not in ordyears[y]:
ordyears[y][lastdateprevmonth] = 0
# make sure there is an entry the first date of this month
firstdateinmonth = datetime(y,thismonth,1)
if firstdateinmonth not in ordyears[y]:
ordyears[y][firstdateinmonth] = 0
# add thisitem to ordered dict for year
ordyears[y][thisitem[0]] = thisitem[1]
return ordyears
def analyzemembership(memberfileh,detailfile=None,overlapfile=None):
#----------------------------------------------------------------------
'''
compare membership statistics, year on year
:param memberfileh: membership file handle, individual records
:param detailfile: optional detailed debug file name
:param overlapfile: optional overlap debug file name to record overlapping join / expiration date periods
:rtype: OrderedDict {year: OrderedDict {datetime:count,...}, ...}
'''
# debug
if detailfile:
_DETL = open(detailfile,'w',newline='')
DETL = csv.DictWriter(_DETL,['ord','effective','name','catchup',
# 'renewal',
'join','expiration'])
DETL.writeheader()
detlrecord = 0
# pull in memberfile
members = RunningAheadMembers(memberfileh,overlapfile=overlapfile)
# iterate through memberships
memberships = members.membership_iter()
## loop through preprocessed records
years = {}
for membership in memberships:
# asc_renewaldate = membership.renew
asc_joindate = membership.join
asc_expdate = membership.expiration
# renewaldate = ymd.asc2dt(asc_renewaldate)
joindate = ymd.asc2dt(asc_joindate)
expdate = ymd.asc2dt(asc_expdate)
fname = membership.fname
lname = membership.lname
dob = membership.dob
fullname = '{}, {}'.format(lname,fname)
# when clicking "Export individual records", joindate is the effective date for the specific year
effectivedate = joindate
## increment the member count for the member's effective date
year = effectivedate.year
# good data starts in 2013
if year >= 2013:
# create year if it hasn't been created
if year not in years:
years[year] = {}
# increment the effectivedate date within the year
years[year][effectivedate] = years[year].get(effectivedate,0) + 1
# debug
if detailfile:
detlrecord += 1
DETL.writerow({'effective':ymd.dt2asc(effectivedate),'name':fullname,
# 'renewal':asc_renewaldate,
'join':asc_joindate,'expiration':asc_expdate,
'ord':detlrecord})
# for all years after effectivedate's until expdate's, increment jan 1
# this happens if there is a grace period and member's renewal counts for following year,
# and for multiyear membership
# NOTE: this is not under "if year >= 2013" because there are some joindates in 2012,
# captured here under jan1, 2013
for y in range(effectivedate.year+1,expdate.year+1):
jan1 = datetime(y,1,1)
if y not in years:
years[y] = {}
years[y][jan1] = years[y].get(jan1,0) + 1
# debug
if detailfile:
detlrecord += 1
DETL.writerow({'effective':ymd.dt2asc(jan1),'name':fullname,
# 'renewal':asc_renewaldate,
'join':asc_joindate,'expiration':asc_expdate,
'catchup':'y',
'ord':detlrecord})
# debug
if detailfile:
_DETL.close()
# create an entry with 0 count for today's date, if none exists (use local time)
# if, say, last entry was July 28, this will have the effect of prettying up the output
# for full month rendering
today = timeu.epoch2dt(time.time()-time.timezone)
thisyear = today.year
thismonth = today.month
thisday = today.day
today = datetime(thisyear, thismonth, thisday) # removes time of day from today
if today not in years[thisyear]:
years[thisyear][today] = 0
# remove any entries for any years accumulated after this year
# this can happen if long expiration dates are in database
for y in range(thisyear+1, max(years.keys())+1):
years.pop(y)
# create orderered dicts
allyears = sorted(list(years.keys()))
ordyears = OrderedDict()
for y in allyears:
ordyears[y] = OrderedDict()
# make sure each month has entry in first and final date, so annotations in rendermembershipanalysis work nicely
thismonth = 1
for thisitem in sorted(list(years[y].items()), key=lambda t: t[0]):
thisdate = thisitem[0]
# if we are at a new month, create an empty entry for the first day of this month
# need while loop in case there was a whole month without memberships
# also make sure there is an entry the last date of the previous month
while (thismonth != thisdate.month):
thismonth += 1
# make sure there is entry the last date of the previous month
prevmonth = thismonth - 1
lastdayprevmonth = monthrange(y,prevmonth)[1]
lastdateprevmonth = datetime(y,thismonth-1,lastdayprevmonth)
if lastdateprevmonth not in ordyears[y]:
ordyears[y][lastdateprevmonth] = 0
# make sure there is an entry the first date of this month
firstdateinmonth = datetime(y,thismonth,1)
if firstdateinmonth not in ordyears[y]:
ordyears[y][firstdateinmonth] = 0
# add thisitem to ordered dict for year
ordyears[y][thisitem[0]] = thisitem[1]
return ordyears
def collect(self, thistask, club_id, searchfile, resultfile, status, begindate=ftime.asc2epoch('1970-01-01'), enddate=ftime.asc2epoch('2999-12-31')):
#----------------------------------------------------------------------
'''
collect race results from a service
:param thistask: this is required for task thistask.update_state()
:param club_id: club id for club being operated on
:param searchfile: path to file containing names, genders, birth dates to search for
:param resultfile: output file path (csv) for detailed results from this service
:param status: dict containing current status
:param begindate: epoch time - choose races between begindate and enddate
:param enddate: epoch time - choose races between begindate and enddate
:param key: key for access to athlinks
'''
# save some parameters as class attributes
self.thistask = thistask
self.status = status
# open files
if type(searchfile) == list:
_IN = searchfile
else:
_IN = open(searchfile,'rb')
IN = csv.DictReader(_IN)
_OUT = open(resultfile,'wb')
OUT = csv.DictWriter(_OUT, self.resultfilehdr)
OUT.writeheader()
try:
# create service
self.openservice(club_id)
# reset begindate to beginning of day, enddate to end of day
dt_begindate = epoch2dt(begindate)
adj_begindate = datetime(dt_begindate.year,dt_begindate.month,dt_begindate.day,0,0,0)
begindate = dt2epoch(adj_begindate)
dt_enddate = epoch2dt(enddate)
adj_enddate = datetime(dt_enddate.year,dt_enddate.month,dt_enddate.day,23,59,59)
enddate = dt2epoch(adj_enddate)
# get start time for debug messaging
start = time.time()
# loop through runners in the input file
for runner in IN:
name = ' '.join([runner['GivenName'],runner['FamilyName']])
dt_dob = ftime.asc2dt(runner['DOB'])
# get results for this athlete
results = self.getresults(name, runner['GivenName'], runner['FamilyName'], runner['Gender'][0], dt_dob, begindate, enddate)
# loop through each result
for result in results:
# protect against bad data, just ignore the result and log the error
try:
outrec = self.convertserviceresult(result)
# maybe user is trying to cancel
except SystemExit:
raise
# otherwise just log and ignore result
except:
app.logger.warning('exception for "{}", result ignored, processing {} result {}\n{}'.format(name, self.servicename, result, traceback.format_exc()))
outrec = None
# only save if service wanted to save
if outrec:
OUT.writerow(outrec)
# update status
status[self.servicename]['lastname'] = name
status[self.servicename]['processed'] += 1
thistask.update_state(state='PROGRESS', meta={'progress':status})
finally:
self.closeservice()
_OUT.close()
if type(searchfile) != list:
_IN.close()
finish = time.time()
app.logger.debug('elapsed time (min) = {}'.format((finish-start)/60))
def collect(searchfile, outfile, begindate, enddate):
#----------------------------------------------------------------------
'''
collect race results from runningahead
:param searchfile: path to file containing names, genders, birth dates to search for
:param outfile: output file path
:param begindate: epoch time - choose races between begindate and enddate
:param enddate: epoch time - choose races between begindate and enddate
'''
outfilehdr = 'GivenName,FamilyName,name,DOB,Gender,race,date,age,miles,km,time'.split(
',')
# open files
_IN = open(searchfile, 'rb')
IN = csv.DictReader(_IN)
_OUT = open(outfile, 'wb')
OUT = csv.DictWriter(_OUT, outfilehdr)
OUT.writeheader()
# common fields between input and output
commonfields = 'GivenName,FamilyName,DOB,Gender'.split(',')
# create runningahead access, grab users who have used the steeplechasers.org portal to RA
ra = runningahead.RunningAhead()
users = ra.listusers()
rausers = []
for user in users:
rauser = ra.getuser(user['token'])
rausers.append((user, rauser))
# reset begindate to beginning of day, enddate to end of day
dt_begindate = timeu.epoch2dt(begindate)
a_begindate = fdate.dt2asc(dt_begindate)
adj_begindate = datetime.datetime(dt_begindate.year, dt_begindate.month,
dt_begindate.day, 0, 0, 0)
e_begindate = timeu.dt2epoch(adj_begindate)
dt_enddate = timeu.epoch2dt(enddate)
a_enddate = fdate.dt2asc(dt_enddate)
adj_enddate = datetime.datetime(dt_enddate.year, dt_enddate.month,
dt_enddate.day, 23, 59, 59)
e_enddate = timeu.dt2epoch(adj_enddate)
# get today's date for high level age filter
start = time.time()
today = timeu.epoch2dt(start)
# loop through runners in the input file
for runner in IN:
fname, lname = runner['GivenName'], runner['FamilyName']
membername = '{} {}'.format(fname, lname)
log.debug('looking for {}'.format(membername))
e_dob = fdate.asc2epoch(runner['DOB'])
dt_dob = fdate.asc2dt(runner['DOB'])
dob = runner['DOB']
gender = runner['Gender'][0]
# find thisuser
foundmember = False
for user, rauser in rausers:
if 'givenName' not in rauser or 'birthDate' not in rauser:
continue # we need to know the name and birth date
givenName = rauser['givenName'] if 'givenName' in rauser else ''
familyName = rauser['familyName'] if 'familyName' in rauser else ''
rausername = '******'.format(givenName, familyName)
if rausername == membername and dt_dob == fdate.asc2dt(
rauser['birthDate']):
foundmember = True
log.debug('found {}'.format(membername))
break
# member is not this ra user, keep looking
# if we couldn't find this member in RA, try the next member
if not foundmember: continue
## skip getting results if participant too young
#todayage = timeu.age(today,dt_dob)
#if todayage < 14: continue
# if we're here, found the right user, now let's look at the workouts
workouts = ra.listworkouts(user['token'],
begindate=a_begindate,
enddate=a_enddate,
getfields=FIELD['workout'].keys())
# save race workouts, if any found
results = []
if workouts:
for wo in workouts:
if wo['workoutName'].lower() != 'race': continue
if 'duration' not in wo['details']:
continue # seen once, not sure why
thisdate = wo['date']
dt_thisdate = fdate.asc2dt(thisdate)
thisdist = runningahead.dist2meters(wo['details']['distance'])
thistime = wo['details']['duration']
thisrace = wo['course']['name'] if wo.has_key(
'course') else 'unknown'
if thistime == 0:
log.warning('{} has 0 time for {} {}'.format(
membername, thisrace, thisdate))
continue
stat = {
'GivenName': fname,
'FamilyName': lname,
'name': membername,
'DOB': dob,
'Gender': gender,
'race': thisrace,
'date': thisdate,
'age': timeu.age(dt_thisdate, dt_dob),
'miles': thisdist / METERSPERMILE,
'km': thisdist / 1000.0,
'time': render.rendertime(thistime, 0)
}
results.append(stat)
# loop through each result
for result in results:
e_racedate = fdate.asc2epoch(result['date'])
# skip result if outside the desired time window
if e_racedate < begindate or e_racedate > enddate: continue
# create output record and copy fields
outrec = result
resulttime = result['time']
# strange case of TicksString = ':00'
if resulttime[0] == ':':
resulttime = '0' + resulttime
while resulttime.count(':') < 2:
resulttime = '0:' + resulttime
outrec['time'] = resulttime
OUT.writerow(outrec)
_OUT.close()
_IN.close()
finish = time.time()
print 'elapsed time (min) = {}'.format((finish - start) / 60)
def summarize(thistask, club_id, sources, status, summaryfile, detailfile, resultsurl, minage=12, minagegrade=20, minraces=3 , mintrend=2, numyears=3, begindate=None, enddate=None):
#----------------------------------------------------------------------
'''
render collected results
:param thistask: this is required for task thistask.update_state()
:param club_id: identifies club for which results are to be stored
:param sources: list of sources / services we're keeping status for
:param summaryfile: summary file name (.csv)
:param detailfile: detail file name (.csv)
:param resultsurl: base url to send results to, for link in summary table
:param minage: minimum age to keep track of stats
:param minagegrade: minimum age grade
:param minraces: minimum races in the same year as enddate
:param mintrend: minimum races over the full period for trendline
:param begindate: render races between begindate and enddate, datetime
:param enddate: render races between begindate and enddate, datetime
'''
# get club slug and location for later
club = Club.query.filter_by(id=club_id).first()
clubslug = club.shname
locsvr = LocationServer()
clublocation = locsvr.getlocation(club.location)
# get maxdistance by service
services = RaceResultService.query.filter_by(club_id=club_id).join(ApiCredentials).all()
maxdistance = {}
for service in services:
attrs = ServiceAttributes(club_id, service.apicredentials.name)
# app.logger.debug('service {} attrs {}'.format(service, attrs.__dict__))
if attrs.maxdistance:
maxdistance[service.apicredentials.name] = attrs.maxdistance
else:
maxdistance[service.apicredentials.name] = None
maxdistance[productname] = None
# set up date range. begindate and enddate take precedence, else use numyears from today
if not (begindate and enddate):
etoday = time.time()
today = timeu.epoch2dt(etoday)
begindate = datetime(today.year-numyears+1,1,1)
enddate = datetime(today.year,12,31)
firstyear = begindate.year
lastyear = enddate.year
yearrange = range(firstyear,lastyear+1)
# get all the requested result data from the database and save in a data structure indexed by runner
## first get the data from the database
results = RaceResult.query.join(Race).join(Runner).filter(RaceResult.club_id==club_id,
Race.date.between(ftime.dt2asc(begindate), ftime.dt2asc(enddate)), Runner.member==True, Runner.active==True).order_by(Runner.lname, Runner.fname).all()
## then set up our status and pass to the front end
for source in sources:
status[source]['status'] = 'summarizing'
status[source]['lastname'] = ''
status[source]['processed'] = 0
status[source]['total'] = sum([1 for result in results if result.source==source])
thistask.update_state(state='PROGRESS', meta={'progress':status})
## prepare to save detail file, for debugging
detlfields = 'runnername,runnerid,dob,gender,resultid,racename,racedate,series,distmiles,distkm,time,timesecs,agpercent,source,sourceid'.split(',')
detailfname = detailfile
_DETL = open(detailfname,'wb')
DETL = csv.DictWriter(_DETL,detlfields)
DETL.writeheader()
## then fill in data structure to hold AnalyzeAgeGrade objects
## use OrderedDict to force aag to be in same order as DETL file, for debugging
aag = collections.OrderedDict()
for result in results:
# skip results which are too far away, if a maxdistance is defined for this source
if maxdistance[result.source]:
locationid = result.race.locationid
if not locationid: continue
racelocation = Location.query.filter_by(id=locationid).first()
distance = get_distance(clublocation, racelocation)
if distance == None or distance > maxdistance[result.source]: continue
thisname = (result.runner.name.lower(), result.runner.dateofbirth)
initaagrunner(aag, thisname, result.runner.fname, result.runner.lname, result.runner.gender, ftime.asc2dt(result.runner.dateofbirth), result.runner.id)
# determine location name. any error gets null string
locationname = ''
if result.race.locationid:
location = Location.query.filter_by(id=result.race.locationid).first()
if location:
locationname = location.name
thisstat = aag[thisname].add_stat(ftime.asc2dt(result.race.date), result.race.distance*METERSPERMILE, result.time, race=result.race.name,
loc=locationname, fuzzyage=result.fuzzyage,
source=result.source, priority=priority[result.source])
### TODO: store result's agpercent, in AgeGrade.crunch() skip agegrade calculation if already present
DETL.writerow(dict(
runnername = result.runner.name,
runnerid = result.runner.id,
dob = result.runner.dateofbirth,
gender = result.runner.gender,
resultid = result.id,
racename = result.race.name,
racedate = result.race.date,
series = result.series.name if result.seriesid else None,
distmiles = result.race.distance,
distkm = result.race.distance*(METERSPERMILE/1000),
timesecs = result.time,
time = rendertime(result.time,0),
agpercent = result.agpercent,
source = result.source,
sourceid = result.sourceid,
))
## close detail file
_DETL.close()
# initialize summary file
summfields = ['name', 'lname', 'fname', 'age', 'gender']
datafields = copy(summfields)
distcategories = ['overall'] + [TRENDLIMITS[tlimit][0] for tlimit in TRENDLIMITS]
datacategories = ['overall'] + [TRENDLIMITS[tlimit][1] for tlimit in TRENDLIMITS]
stattypes = ['1yr agegrade','avg agegrade','trend','numraces','stderr','r-squared','pvalue']
statdatatypes = ['1yr-agegrade','avg-agegrade','trend','numraces','stderr','r-squared','pvalue']
for stattype, statdatatype in zip(stattypes, statdatatypes):
for distcategory, datacategory in zip(distcategories, datacategories):
summfields.append('{}\n{}'.format(stattype, distcategory))
datafields.append('{}-{}'.format(statdatatype, datacategory))
if stattype == 'numraces':
for year in yearrange:
summfields.append('{}\n{}'.format(stattype, year))
datafields.append('{}-{}'.format(statdatatype, lastyear-year))
# save summary file columns for resultsanalysissummary
dtcolumns = json.dumps([{ 'data':d, 'name':d, 'label':l } for d,l in zip(datafields, summfields)])
columnsfilename = summaryfile + '.cols'
with open(columnsfilename, 'w') as cols:
cols.write(dtcolumns)
# set up summary file
summaryfname = summaryfile
_SUMM = open(summaryfname,'wb')
SUMM = csv.DictWriter(_SUMM,summfields)
SUMM.writeheader()
# loop through each member we've recorded information about
for thisname in aag:
fullname, fname, lname, gender, dob, runnerid = aag[thisname].get_runner()
rendername = fullname.title()
# check stats before deduplicating
statcount = {}
stats = aag[thisname].get_stats()
for source in sources:
statcount[source] = sum([1 for s in stats if s.source == source])
# remove duplicate entries
aag[thisname].deduplicate()
# crunch the numbers
aag[thisname].crunch() # calculate age grade for each result
stats = aag[thisname].get_stats()
jan1 = ftime.asc2dt('{}-1-1'.format(lastyear))
runnerage = timeu.age(jan1, dob)
# filter out runners younger than allowed
if runnerage < minage: continue
# filter out runners who have not run enough races
stats = aag[thisname].get_stats()
if enddate:
lastyear = enddate.year
else:
lastyear = timeu.epoch2dt(time.time()).year
lastyearstats = [s for s in stats if s.date.year==lastyear]
if len(lastyearstats) < minraces: continue
# fill in row for summary output
summout = {}
# get link for this runner's results chart
# see http://stackoverflow.com/questions/2506379/add-params-to-given-url-in-python
url_parts = list(urlparse(resultsurl))
query = dict(parse_qsl(url_parts[4]))
query.update({'club': clubslug, 'runnerid': runnerid, 'begindate': ftime.dt2asc(begindate), 'enddate': ftime.dt2asc(enddate)})
url_parts[4] = urlencode(query)
resultslink = urlunparse(url_parts)
summout['name'] = '<a href={} target=_blank>{}</a>'.format(resultslink, rendername)
summout['fname'] = fname
summout['lname'] = lname
summout['age'] = runnerage
summout['gender'] = gender
# set up to collect averages
avg = collections.OrderedDict()
# draw trendlines, write output
allstats = aag[thisname].get_stats()
if len(allstats) > 0:
avg['overall'] = mean([s.ag for s in allstats])
trend = aag[thisname].get_trendline()
oneyrstats = [s.ag for s in allstats if s.date.year == lastyear]
if len(oneyrstats) > 0:
summout['1yr agegrade\noverall'] = mean(oneyrstats)
if len(allstats) > 0:
summout['avg agegrade\noverall'] = avg['overall']
if len(allstats) >= mintrend and allstats[0].date != allstats[-1].date:
summout['trend\noverall'] = trend.improvement
summout['stderr\noverall'] = trend.stderr
summout['r-squared\noverall'] = trend.r2**2
summout['pvalue\noverall'] = trend.pvalue
summout['numraces\noverall'] = len(allstats)
for year in yearrange:
summout['numraces\n{}'.format(year)] = len([s for s in allstats if s.date.year==year])
for tlimit in TRENDLIMITS:
distcategory,distcolor = TRENDLIMITS[tlimit]
tstats = [s for s in allstats if s.dist >= tlimit[0] and s.dist < tlimit[1]]
if len(tstats) > 0:
avg[distcategory] = mean([s.ag for s in tstats])
summout['avg agegrade\n{}'.format(distcategory)] = avg[distcategory]
summout['numraces\n{}'.format(distcategory)] = len(tstats)
oneyrcategory = [s.ag for s in tstats if s.date.year == lastyear]
if len(oneyrcategory) > 0:
summout['1yr agegrade\n{}'.format(distcategory)] = mean(oneyrcategory)
if len(tstats) >= mintrend and tstats[0].date != tstats[-1].date:
try:
trend = aag[thisname].get_trendline(thesestats=tstats)
except ZeroDivisionError:
app.logger.debug('ZeroDivisionError - processing {}'.format(rendername))
trend = None
# ignore trends which can't be calculated
if trend:
summout['trend\n{}'.format(distcategory)] = trend.improvement
summout['stderr\n{}'.format(distcategory)] = trend.stderr
summout['r-squared\n{}'.format(distcategory)] = trend.r2
summout['pvalue\n{}'.format(distcategory)] = trend.pvalue
SUMM.writerow(summout)
# update status
for source in sources:
status[source]['processed'] += statcount[source]
status[source]['lastname'] = rendername
thistask.update_state(state='PROGRESS', meta={'progress':status})
_SUMM.close()
def getresults(self, name, fname, lname, gender, dt_dob, begindate, enddate):
#----------------------------------------------------------------------
'''
retrieves a list of results for a single name
must be overridden when ResultsCollect is instantiated
use dt_dob to filter errant race results, based on age of runner on race day
:param name: name of participant for which results are to be returned
:param fname: first name of participant
:param lname: last name of participant
:param gender: 'M' or 'F'
:param dt_dob: participant's date of birth, as datetime
:param begindate: epoch time for start of results, 00:00:00 on date to begin
:param end: epoch time for end of results, 23:59:59 on date to finish
:rtype: list of serviceresults, each of which can be processed by convertresult
'''
# remember participant data
self.name = name
self.fname = fname
self.lname = lname
self.gender = gender
self.dt_dob = dt_dob
self.dob = ftime.dt2asc(dt_dob)
# get results for this athlete
allresults = self.service.listathleteresults(name)
# filter by date and by age
filteredresults = []
for result in allresults:
e_racedate = athlinks.gettime(result['Race']['RaceDate'])
# skip result if outside the desired time window
if e_racedate < begindate or e_racedate > enddate: continue
# skip result if wrong gender
resultgen = result['Gender'][0]
if resultgen != gender: continue
# skip result if runner's age doesn't match the age within the result
# sometimes athlinks stores the age group of the runner, not exact age,
# so also check if this runner's age is within the age group, and indicate if so
dt_racedate = timeu.epoch2dt(e_racedate)
racedateage = timeu.age(dt_racedate,dt_dob)
resultage = int(result['Age'])
result['fuzzyage'] = False
if resultage != racedateage:
# if results are not stored as age group, skip this result
if (resultage/5)*5 != resultage:
continue
# result's age might be age group, not exact age
else:
# if runner's age consistent with race age, use result, but mark "fuzzy"
if (racedateage/5)*5 == resultage:
result['fuzzyage'] = True
# otherwise skip result
else:
continue
# if we reach here, the result is ok, and is added to filteredresults
filteredresults.append(result)
# back to caller
return filteredresults
def main():
#----------------------------------------------------------------------
'''
update club membership information
'''
parser = argparse.ArgumentParser(
version='{0} {1}'.format('runningclub', version.__version__))
parser.add_argument('memberfile',
help='csv, xls or xlsx file with member information')
parser.add_argument(
'-r',
'--racedb',
help=
'filename of race database (default is as configured during rcuserconfig)',
default=None)
parser.add_argument('--debug',
help='if set, create updatemembers.txt for debugging',
action='store_true')
args = parser.parse_args()
OUT = None
if args.debug:
OUT = open('updatemembers.txt', 'w')
racedb.setracedb(args.racedb)
session = racedb.Session()
# get clubmembers from file
memberfile = args.memberfile
root, ext = os.path.splitext(memberfile)
if ext in ['.xls', '.xlsx']:
members = clubmember.XlClubMember(memberfile)
elif ext in ['.csv']:
members = clubmember.CsvClubMember(memberfile)
else:
print(
'***ERROR: invalid memberfile {}, must be csv, xls or xlsx'.format(
memberfile))
return
# get old clubmembers from database
dbmembers = clubmember.DbClubMember() # use default database
# get all the member runners currently in the database
# hash them into dict by (name,dateofbirth)
allrunners = session.query(racedb.Runner).filter_by(member=True,
active=True).all()
inactiverunners = {}
for thisrunner in allrunners:
inactiverunners[thisrunner.name, thisrunner.dateofbirth] = thisrunner
if OUT:
OUT.write('found id={0}, runner={1}\n'.format(
thisrunner.id, thisrunner))
# make report for new members found with this memberfile
logdir = os.path.dirname(args.memberfile)
memberfilebase = os.path.splitext(os.path.basename(args.memberfile))[0]
newmemlogname = '{0}-newmem.csv'.format(memberfilebase)
NEWMEM = open(os.path.join(logdir, newmemlogname), 'w', newline='')
NEWMEMCSV = csv.DictWriter(NEWMEM, ['name', 'dob'])
NEWMEMCSV.writeheader()
# prepare for age check
thisyear = timeu.epoch2dt(time.time()).year
asofasc = '{}-1-1'.format(thisyear) # jan 1 of current year
asof = tYmd.asc2dt(asofasc)
# process each name in new membership list
allmembers = members.getmembers()
for name in allmembers:
thesemembers = allmembers[name]
# NOTE: may be multiple members with same name
for thismember in thesemembers:
thisname = thismember['name']
thisdob = thismember['dob']
thisgender = thismember['gender'][0].upper(
) # male -> M, female -> F
thishometown = thismember['hometown']
# prep for if .. elif below by running some queries
# handle close matches, if DOB does match
age = timeu.age(asof, tYmd.asc2dt(thisdob))
matchingmember = dbmembers.findmember(thisname, age, asofasc)
dbmember = None
if matchingmember:
membername, memberdob = matchingmember
if memberdob == thisdob:
dbmember = racedb.getunique(session,
racedb.Runner,
member=True,
name=membername,
dateofbirth=thisdob)
# TODO: need to handle case where dob transitions from '' to actual date of birth
# no member found, maybe there is nonmember of same name already in database
if dbmember is None:
dbnonmember = racedb.getunique(session,
racedb.Runner,
member=False,
name=thisname)
# TODO: there's a slim possibility that there are two nonmembers with the same name, but I'm sure we've already
# bolloxed that up in importresult as there's no way to discriminate between the two
# make report for new members
NEWMEMCSV.writerow({'name': thisname, 'dob': thisdob})
# see if this runner is a member in the database already, or was a member once and make the update
# add or update runner in database
# get instance, if it exists, and make any updates
found = False
if dbmember is not None:
thisrunner = racedb.Runner(membername, thisdob, thisgender,
thishometown)
# this is also done down below, but must be done here in case member's name has changed
if (thisrunner.name,
thisrunner.dateofbirth) in inactiverunners:
inactiverunners.pop(
(thisrunner.name, thisrunner.dateofbirth))
# overwrite member's name if necessary
thisrunner.name = thisname
added = racedb.update(session,
racedb.Runner,
dbmember,
thisrunner,
skipcolumns=['id'])
found = True
# if runner's name is in database, but not a member, see if this runner is a nonmemember which can be converted
# Check first result for age against age within the input file
# if ages match, convert nonmember to member
elif dbnonmember is not None:
# get dt for date of birth, if specified
try:
dob = tYmd.asc2dt(thisdob)
except ValueError:
dob = None
# nonmember came into the database due to a nonmember race result, so we can use any race result to check nonmember's age
if dob:
result = session.query(racedb.RaceResult).filter_by(
runnerid=dbnonmember.id).first()
resultage = result.agage
racedate = tYmd.asc2dt(result.race.date)
expectedage = racedate.year - dob.year - int(
(racedate.month, racedate.day) < (dob.month, dob.day))
# we found the right person, always if dob isn't specified, but preferably check race result for correct age
if dob is None or resultage == expectedage:
thisrunner = racedb.Runner(thisname, thisdob, thisgender,
thishometown)
added = racedb.update(session,
racedb.Runner,
dbnonmember,
thisrunner,
skipcolumns=['id'])
found = True
else:
print(
'{} found in database, wrong age, expected {} found {} in {}'
.format(thisname, expectedage, resultage, result))
# TODO: need to make file for these, also need way to force update, because maybe bad date in database for result
# currently this will cause a new runner entry
# if runner was not found in database, just insert new runner
if not found:
thisrunner = racedb.Runner(thisname, thisdob, thisgender,
thishometown)
added = racedb.insert_or_update(session,
racedb.Runner,
thisrunner,
skipcolumns=['id'],
name=thisname,
dateofbirth=thisdob)
# remove this runner from collection of runners which should be deactivated in database
if (thisrunner.name, thisrunner.dateofbirth) in inactiverunners:
inactiverunners.pop((thisrunner.name, thisrunner.dateofbirth))
if OUT:
if added:
OUT.write('added or updated {0}\n'.format(thisrunner))
else:
OUT.write('no updates necessary {0}\n'.format(thisrunner))
# any runners remaining in 'inactiverunners' should be deactivated
for (name, dateofbirth) in inactiverunners:
thisrunner = session.query(
racedb.Runner).filter_by(name=name, dateofbirth=dateofbirth).first(
) # should be only one returned by filter
thisrunner.active = False
if OUT:
OUT.write('deactivated {0}\n'.format(thisrunner))
session.commit()
session.close()
NEWMEM.close()
if OUT:
OUT.close()
def convertserviceresult(self, result):
#----------------------------------------------------------------------
'''
converts a single service result to dict suitable to be saved in resultfile
result must be converted to dict with keys in `resultfilehdr` provided at instance creation
must be overridden when ResultsCollect is instantiated
use return value of None for cases when results could not be filtered by `:meth:getresults`
:param fname: participant's first name
:param lname: participant's last name
:param result: single service result, from list retrieved through `getresults`
:rtype: dict with keys matching `resultfilehdr`, or None if result is not to be saved
'''
# create output record and copy common fields
outrec = {}
# copy participant information
outrec['name'] = self.name
outrec['GivenName'] = self.fname
outrec['FamilyName'] = self.lname
outrec['DOB'] = self.dob
outrec['Gender'] = self.gender
# some debug items - assume everything is cached
coursecached = True
racecached = True
# get course used for this result
courseid = '{}/{}'.format(result['Race']['RaceID'], result['CourseID'])
course = Course.query.filter_by(club_id=self.club_id, source='athlinks', sourceid=courseid).first()
# cache course if not done already
race = None
if not course:
coursecached = False
coursedata = self.service.getcourse(result['Race']['RaceID'], result['CourseID'])
distmiles = athlinks.dist2miles(coursedata['Courses'][0]['DistUnit'],coursedata['Courses'][0]['DistTypeID'])
distkm = athlinks.dist2km(coursedata['Courses'][0]['DistUnit'],coursedata['Courses'][0]['DistTypeID'])
if distkm < 0.050: return None # skip timed events, which seem to be recorded with 0 distance
# skip result if not Running or Trail Running race
thiscategory = coursedata['Courses'][0]['RaceCatID']
if thiscategory not in race_category: return None
course = Course()
course.club_id = self.club_id
course.source = 'athlinks'
course.sourceid = courseid
# strip racename and coursename here to make sure detail file matches what is stored in database
racename = csvu.unicode2ascii(coursedata['RaceName']).strip()
coursename = csvu.unicode2ascii(coursedata['Courses'][0]['CourseName']).strip()
course.name = '{} / {}'.format(racename,coursename)
# maybe truncate to FIRST part of race name
if len(course.name) > MAX_RACENAME_LEN:
course.name = course.name[:MAX_RACENAME_LEN]
course.date = ftime.epoch2asc(athlinks.gettime(coursedata['RaceDate']))
course.location = csvu.unicode2ascii(coursedata['Home'])
# maybe truncate to LAST part of location name, to keep most relevant information (state, country)
if len(course.location) > MAX_LOCATION_LEN:
course.location = course.location[-MAX_LOCATION_LEN:]
# TODO: adjust marathon and half marathon distances?
course.distkm =distkm
course.distmiles = distmiles
course.surface = race_category[thiscategory]
# retrieve or add race
# flush should allow subsequent query per http://stackoverflow.com/questions/4201455/sqlalchemy-whats-the-difference-between-flush-and-commit
# Race has uniqueconstraint for club_id/name/year/fixeddist. It's been seen that there are additional races in athlinks,
# but just assume the first is the correct one.
raceyear = ftime.asc2dt(course.date).year
race = Race.query.filter_by(club_id=self.club_id, name=course.name, year=raceyear, fixeddist=race_fixeddist(course.distmiles)).first()
### TODO: should the above be .all() then check for first race within epsilon distance?
if not race:
racecached = False
race = Race(self.club_id, raceyear)
race.name = course.name
race.distance = course.distmiles
race.fixeddist = race_fixeddist(race.distance)
race.date = course.date
race.active = True
race.external = True
race.surface = course.surface
loc = self.locsvr.getlocation(course.location)
race.locationid = loc.id
db.session.add(race)
db.session.flush() # force id to be created
course.raceid = race.id
db.session.add(course)
db.session.flush() # force id to be created
# maybe course was cached but location of race wasn't
# update location of result race, if needed, and if supplied
# this is here to clean up old database data
if not race:
race = Race.query.filter_by(club_id=self.club_id, name=course.name, year=ftime.asc2dt(course.date).year, fixeddist=race_fixeddist(course.distmiles)).first()
if not race.locationid and course.location:
# app.logger.debug('updating race with location {}'.format(course.location))
loc = self.locsvr.getlocation(course.location)
race.locationid = loc.id
insert_or_update(db.session, Race, race, skipcolumns=['id'],
club_id=self.club_id, name=course.name, year=ftime.asc2dt(course.date).year, fixeddist=race_fixeddist(course.distmiles))
# else:
# app.logger.debug('race.locationid={} course.location="{}"'.format(race.locationid, course.location))
# debug races
if self.racefile:
racestatusl = []
if not coursecached: racestatusl.append('addcourse')
if not racecached: racestatusl.append('addrace')
if not racestatusl: racestatusl.append('cached')
racestatus = '-'.join(racestatusl)
racerow = {'status': racestatus, 'runner': self.name}
for racefield in self.racefields:
if racefield in ['status', 'runner']: continue
racerow[racefield] = getattr(course,racefield)
self.RACE.writerow(racerow)
# fill in output record fields from result, course
# combine name, get age
outrec['age'] = result['Age']
outrec['fuzzyage'] = result['fuzzyage']
# leave athlid blank if result not from an athlink member
athlmember = result['IsMember']
if athlmember:
outrec['athlid'] = result['RacerID']
# remember the entryid, high water mark of which can be used to limit the work here
outrec['entryid'] = result['EntryID']
# race name, location; convert from unicode if necessary
# TODO: make function to do unicode translation -- apply to runner name as well (or should csv just store unicode?)
outrec['race'] = course.name
outrec['date'] = course.date
outrec['loc'] = course.location
outrec['miles'] = course.distmiles
outrec['km'] = course.distkm
outrec['category'] = course.surface
resulttime = result['TicksString']
# strange case of TicksString = ':00'
if resulttime[0] == ':':
resulttime = '0'+resulttime
while resulttime.count(':') < 2:
resulttime = '0:'+resulttime
outrec['time'] = resulttime
# strange case of 0 time, causes ZeroDivisionError and is clearly not valid
if timeu.timesecs(resulttime) == 0: return None
# leave out age grade if exception occurs, skip results which have outliers
try:
# skip result if runner's age doesn't match the age within the result
# sometimes athlinks stores the age group of the runner, not exact age,
# so also check if this runner's age is within the age group, and indicate if so
e_racedate = athlinks.gettime(result['Race']['RaceDate'])
resultgen = result['Gender'][0]
dt_racedate = timeu.epoch2dt(e_racedate)
racedateage = timeu.age(dt_racedate,self.dt_dob)
agpercent,agresult,agfactor = ag.agegrade(racedateage,resultgen,course.distmiles,timeu.timesecs(resulttime))
outrec['ag'] = agpercent
if agpercent < 15 or agpercent >= 100: return None # skip obvious outliers
except:
app.logger.warning(traceback.format_exc())
pass
# and we're done
return outrec
def collect(searchfile,outfile,begindate,enddate):
#----------------------------------------------------------------------
'''
collect race results from ultrasignup
:param searchfile: path to file containing names, genders, birth dates to search for
:param outfile: output file path
:param begindate: epoch time - choose races between begindate and enddate
:param enddate: epoch time - choose races between begindate and enddate
'''
# open files
_IN = open(searchfile,'rb')
IN = csv.DictReader(_IN)
_OUT = open(outfile,'wb')
OUT = csv.DictWriter(_OUT,UltraSignupResultFile.filehdr)
OUT.writeheader()
# common fields between input and output
commonfields = 'GivenName,FamilyName,DOB,Gender'.split(',')
# create ultrasignup access
ultra = ultrasignup.UltraSignup(debug=True)
# reset begindate to beginning of day, enddate to end of day
dt_begindate = timeu.epoch2dt(begindate)
adj_begindate = datetime.datetime(dt_begindate.year,dt_begindate.month,dt_begindate.day,0,0,0)
begindate = timeu.dt2epoch(adj_begindate)
dt_enddate = timeu.epoch2dt(enddate)
adj_enddate = datetime.datetime(dt_enddate.year,dt_enddate.month,dt_enddate.day,23,59,59)
enddate = timeu.dt2epoch(adj_enddate)
# get today's date for high level age filter
start = time.time()
today = timeu.epoch2dt(start)
# loop through runners in the input file
for runner in IN:
fname,lname = runner['GivenName'],runner['FamilyName']
e_dob = ftime.asc2epoch(runner['DOB'])
dt_dob = ftime.asc2dt(runner['DOB'])
gender = runner['Gender'][0]
## skip getting results if participant too young
#todayage = timeu.age(today,dt_dob)
#if todayage < 14: continue
# get results for this athlete
results = ultra.listresults(fname,lname)
# loop through each result
for result in results:
e_racedate = ftime.asc2epoch(result.racedate)
# skip result if outside the desired time window
if e_racedate < begindate or e_racedate > enddate: continue
# skip result if runner's age doesn't match the age within the result
dt_racedate = timeu.epoch2dt(e_racedate)
racedateage = timeu.age(dt_racedate,dt_dob)
if result.age != racedateage: continue
# skip result if runner's gender doesn't match gender within the result
resultgen = result.gender
if resultgen != runner['Gender'][0]: continue
# create output record and copy common fields
outrec = {}
for field in commonfields:
outrec[field] = runner[field]
# fill in output record fields from runner, result
# combine name, get age
outrec['name'] = '{} {}'.format(runner['GivenName'],runner['FamilyName'])
outrec['age'] = result.age
# race name, location; convert from unicode if necessary
racename = result.racename
outrec['race'] = racename
outrec['date'] = ftime.epoch2asc(e_racedate)
outrec['loc'] = '{}, {}'.format(result.racecity, result.racestate)
# distance, category, time
distmiles = result.distmiles
distkm = result.distkm
if distkm is None or distkm < 0.050: continue # should already be filtered within ultrasignup, but just in case
outrec['miles'] = distmiles
outrec['km'] = distkm
resulttime = result.racetime
# int resulttime means DNF, most likely -- skip this result
if type(resulttime) == int: continue
# strange case of TicksString = ':00'
if resulttime[0] == ':':
resulttime = '0'+resulttime
while resulttime.count(':') < 2:
resulttime = '0:'+resulttime
outrec['time'] = resulttime
# just leave out age grade if exception occurs
try:
agpercent,agresult,agfactor = ag.agegrade(racedateage,gender,distmiles,timeu.timesecs(resulttime))
outrec['ag'] = agpercent
if agpercent < 15 or agpercent >= 100: continue # skip obvious outliers
except:
pass
OUT.writerow(outrec)
_OUT.close()
_IN.close()
finish = time.time()
print 'number of URLs retrieved = {}'.format(ultra.geturlcount())
print 'elapsed time (min) = {}'.format((finish-start)/60)
def main():
#----------------------------------------------------------------------
parser = argparse.ArgumentParser(version='{0} {1}'.format('running',version.__version__))
parser.add_argument('infile',help='file generated by racewx')
parser.add_argument('racename',help='race name')
args = parser.parse_args()
infile = args.infile
racename = args.racename
# get input
_WX = open(infile,'rb')
WX = csv.DictReader(_WX)
wxdata = []
for wx in WX:
wxdata.append(wx)
_WX.close()
# for now, filter out all but the max 'exectime' entries
lastexec = max([int(wx['exectime']) for wx in wxdata])
while int(wxdata[0]['exectime']) != lastexec:
wxdata.pop(0)
# pull out fields to plot
wxplot = {}
plotfields = ['time','temperature','windchill','heatindex','dewpoint','windSpeed','windBearing','cloudCover','precipProbability','precipIntensity','cloudCover']
for f in plotfields:
wxplot[f] = [float(wx[f]) if wx[f]!='' else None for wx in wxdata]
# get range on 30 minute boundaries
starttime = int(wxplot['time'][0])
fintime = int(wxplot['time'][-1])
adjstart = (starttime / (30*60)) * (30*60) # rounds to next lowest 30 minute boundary
adjfin = ((fintime-1 + 30*60) / (30*60)) * (30*60) # rounds to next highest 30 minute boundary
startdt = timeu.epoch2dt(adjstart)
findt = timeu.epoch2dt(adjfin)
# time zone stuff, based on starting point
lat = float(wxdata[0]['lat'])
lon = float(wxdata[0]['lon'])
tzid = racewx.gettzid(lat,lon)
tz = pytz.timezone(tzid)
wxplot['localtime'] = [timeu.utcdt2tzdt(timeu.epoch2dt(tm),tzid) for tm in wxplot['time']]
# plot data
fig = plt.figure()
ttitle = timeu.asctime('%m/%d/%Y')
racedate = ttitle.epoch2asc(wxplot['time'][0])
fdate = ttitle.epoch2asc(lastexec)
fig.suptitle('forecast for {race} {date}\nforecast date {fdate}\nPowered by Forecast.io'.format(race=racename,date=racedate,fdate=fdate),size='small')
# set some formatting parameters
lw = 0.5 # line width
windcolor = 'b'
legendx = 1.35
# plot control
exists = {}
for f in ['windchill','heatindex']:
exists[f] = len([it for it in wxplot[f] if it is not None]) != 0
for f in ['precipIntensity']:
exists[f] = len([it for it in wxplot[f] if it > 0.0]) != 0
# plot temperatures
ax1 = fig.add_subplot(311)
ax1.plot(wxplot['localtime'],wxplot['temperature'],'k-',label='temperature', linewidth=lw)
if exists['windchill']:
ax1.plot(wxplot['localtime'],wxplot['windchill'],'b-',label='wind chill', linewidth=lw)
if exists['heatindex']:
ax1.plot(wxplot['localtime'],wxplot['heatindex'],'r-',label='heat index', linewidth=lw)
ax1.plot(wxplot['localtime'],wxplot['dewpoint'],'g-',label='dew point', linewidth=lw)
ax1.set_xlim(startdt,findt)
fig.subplots_adjust(top=0.88,right=0.75,bottom=0.15)
hfmt = dates.DateFormatter('%H:%M',tz=tz)
ax1.xaxis.set_major_formatter(hfmt)
ax1.xaxis.set_major_locator(dates.MinuteLocator(interval=30))
ax1.grid('on')
plt.setp(ax1.get_xticklabels(), visible=False)
plt.setp(ax1.get_yticklabels(), fontsize='small')
ax1.set_ylabel('degrees \nFahrenheit', fontsize='small')
#font = fm.FontProperties(fname='Humor-Sans.ttf')
font = fm.FontProperties()
xsmallfont = copy.deepcopy(font)
xsmallfont.set_size('x-small')
ax1.legend(prop=xsmallfont,loc='upper right', bbox_to_anchor=(legendx, 1))
# plot wind
ax2 = fig.add_subplot(312)
ax2.plot(wxplot['localtime'],wxplot['windSpeed'],label='wind speed', linewidth=lw, color=windcolor)
# note polar-> rectangular flips x,y from standard transformation because theta is from North instead of East
# not sure why need to invert U and V to get barb to point in right direction. Maybe vector comes from U,V and points to origin?
U = [-1*wxplot['windSpeed'][i]*math.sin(math.radians(wxplot['windBearing'][i])) for i in range(len(wxplot['windSpeed']))]
V = [-1*wxplot['windSpeed'][i]*math.cos(math.radians(wxplot['windBearing'][i])) for i in range(len(wxplot['windSpeed']))]
xdates = dates.date2num(wxplot['localtime']) # barbs requires floats, not datetime
ax2.barbs(xdates,wxplot['windSpeed'], U, V, length=5, barbcolor=windcolor, flagcolor=windcolor, linewidth=lw)
ax2.set_xlim(dates.date2num(startdt),dates.date2num(findt))
miny,maxy = ax2.get_ylim()
ax2.set_ylim(round(miny*0.8),round(maxy*1.2))
ax2.xaxis.set_major_formatter(hfmt)
ax2.xaxis.set_major_locator(dates.MinuteLocator(interval=30))
ax2.grid('on')
#plt.setp(ax2.get_xticklabels(), rotation='vertical', fontsize='small')
plt.setp(ax2.get_xticklabels(), visible=False)
plt.setp(ax2.get_yticklabels(), fontsize='small')
ax2.set_ylabel('miles per hour', fontsize='small')
ax2.legend(prop=xsmallfont,loc='upper right', bbox_to_anchor=(legendx, 1))
ax3 = fig.add_subplot(313)
precipprob = [100*(prob or 0) for prob in wxplot['precipProbability']]
cloudcover = [100*(cover or 0) for cover in wxplot['cloudCover']]
ax3.plot(wxplot['localtime'],precipprob,label='rain probability', linewidth=lw, color='b')
ax3.plot(wxplot['localtime'],cloudcover,label='cloud cover', linewidth=lw, color='g')
ax3.set_ylabel('percent', fontsize='small')
ax3.set_xlim(dates.date2num(startdt),dates.date2num(findt))
ax3.xaxis.set_major_formatter(hfmt)
ax3.xaxis.set_major_locator(dates.MinuteLocator(interval=30))
ax3.grid('on')
ax3.set_ylim(0,100)
plt.setp(ax3.get_xticklabels(), rotation='vertical', fontsize='small')
plt.setp(ax3.get_yticklabels(), fontsize='small')
ax3.legend(prop=xsmallfont,loc='upper right', bbox_to_anchor=(legendx, 1.1))
if exists['precipIntensity']:
ax4 = ax3.twinx()
#ax4.plot(wxplot['localtime'],wxplot['precipIntensity'],label='intensity', linewidth=lw, color='r')
#ax4.set_yscale('log')
ax4.semilogy(wxplot['localtime'],wxplot['precipIntensity'],label='intensity', nonposy='mask',linewidth=lw, color='r')
#ax4.set_ylabel('precipitation 0.002 very light sprinkling, 0.017 light precipitation, 0.1 precipitation, and 0.4 very heavy precipitation')
ax4.set_ylabel('intensity',fontsize='small')
ax4.set_ylim(0,0.5)
plt.setp(ax4.get_yticklabels(), fontsize='small')
ax4.legend(prop=xsmallfont,loc='upper right', bbox_to_anchor=(legendx, 0.75))
tfile = timeu.asctime('%Y-%m-%d')
fdate = tfile.epoch2asc(lastexec)
racename = re.sub('\s','',racename) # remove whitespace
outfile = 'race-weather-{race}-{fdate}.png'.format(race=racename,fdate=fdate)
fig.savefig(outfile,format='png')