def getconfig(configfile):
#----------------------------------------------------------------------
'''
get configuratoin from configfile
:param configfile: file containing configuration, formatted per
python ConfigParser
:rtype: dict with configuration key, value pairs
'''
# combine app and runsignup sections
config = getitems(configfile, 'app')
config.update(getitems(configfile, 'runsignup'))
return config
def summarize(configfile, debug=False):
#----------------------------------------------------------------------
'''
Summarize the membership stats and members for a given runsignup club.
:param configfile: configuration filename
:param debug: True for requests debugging
'''
# configuration file supplied -- pull credentials from the app section
appconfig = getitems(configfile, 'runsignup')
club = appconfig['RSU_CLUB']
membercachefile = appconfig['RSU_CACHEFILE']
memberstatsfile = appconfig['RSU_STATSFILE']
key = appconfig['RSU_KEY']
secret = appconfig['RSU_SECRET']
# update member cache file
# note this update is done under lock to prevent any apache thread from disrupting it
members = updatemembercache(club,
membercachefile,
key=key,
secret=secret,
debug=debug)
# analyze the memberships
memberstats = analyzemembership(membercachefile, statsfile=memberstatsfile)
# for debugging
return members, memberstats
def __init__(self, configfiles):
if type(configfiles) == str:
configfiles = [configfiles]
# connect to database based on configuration
config = {}
for configfile in configfiles:
config.update(getitems(configfile, 'database'))
dbuser = config['dbuser']
password = config['dbpassword']
dbserver = config['dbserver']
dbname = config['dbname']
# app.logger.debug('using mysql://{uname}:*******@{server}/{dbname}'.format(uname=dbuser,server=dbserver,dbname=dbname))
db_uri = 'mysql://{uname}:{pw}@{server}/{dbname}'.format(
uname=dbuser, pw=password, server=dbserver, dbname=dbname)
self.SQLALCHEMY_DATABASE_URI = db_uri
# https://flask-sqlalchemy.palletsprojects.com/en/2.x/binds/
userdbuser = config['userdbuser']
userpassword = config['userdbpassword']
userdbserver = config['userdbserver']
userdbname = config['userdbname']
userdb_uri = 'mysql://{uname}:{pw}@{server}/{dbname}'.format(
uname=userdbuser,
pw=userpassword,
server=userdbserver,
dbname=userdbname)
self.SQLALCHEMY_BINDS = {'users': userdb_uri}
def summarize(configfile, debug=False):
#----------------------------------------------------------------------
'''
Summarize the membership stats and members for a given runsignup club.
:param configfile: configuration filename
:param debug: True for requests debugging
'''
# configuration file supplied -- pull credentials from the app section
appconfig = getitems(configfile, 'runsignup')
club = appconfig['RSU_CLUB']
membercachefile = appconfig['RSU_CACHEFILE']
memberstatsfile = appconfig['RSU_STATSFILE']
key = appconfig['RSU_KEY']
secret = appconfig['RSU_SECRET']
# update member cache file
# note this update is done under lock to prevent any apache thread from disrupting it
members = updatemembercache(club, membercachefile, key=key, secret=secret, debug=debug)
# analyze the memberships
memberstats = analyzemembership(membercachefile, statsfile=memberstatsfile)
# for debugging
return members, memberstats
def __init__(self, configpath):
# connect to database based on configuration
config = getitems(configpath, 'database')
dbuser = config['dbuser']
password = config['dbpassword']
dbserver = config['dbserver']
dbname = config['dbname']
# app.logger.debug('using mysql://{uname}:*******@{server}/{dbname}'.format(uname=dbuser,server=dbserver,dbname=dbname))
db_uri = 'mysql://{uname}:{pw}@{server}/{dbname}'.format(
uname=dbuser, pw=password, server=dbserver, dbname=dbname)
self.SQLALCHEMY_DATABASE_URI = db_uri
def __init__(self, configfiles):
if type(configfiles) == str:
configfiles = [configfiles]
# connect to database based on configuration
config = {}
for configfile in configfiles:
config.update(getitems(configfile, 'database'))
dbuser = config['dbuser']
password = config['dbpassword']
dbserver = config['dbserver']
dbname = config['dbname']
db_uri = 'mysql://{uname}:{pw}@{server}/{dbname}'.format(
uname=dbuser, pw=password, server=dbserver, dbname=dbname)
self.SQLALCHEMY_DATABASE_URI = db_uri
def summarize(club, memberstatsfile, membersummaryfile, membershipfile=None, membercachefilename=None, update=False, debug=False, configfile=None):
#----------------------------------------------------------------------
'''
Summarize the membership stats and members for a given RunningAHEAD club.
If membershipfile is not supplied, retrieve the member data from RunningAHEAD
using the priviledged user token.
:param club: club slug for RunningAHEAD
:param memberstatsfile: output json file with member statistics
:param membersummaryfile: output csv file with summarized member information
:param membershipfile: filename, file handle, or list with member data (optional)
:param membercachefilename: name of optional file to cache detailed member data
:param update: True if cache should be updated
:param debug: True for requests debugging
:param configfile: optional configuration filename
'''
# configuration file supplied -- pull credentials from the app section
if configfile:
from loutilities.configparser import getitems
appconfig = getitems(configfile, 'app')
raprivuser = appconfig['RAPRIVUSER']
rakey = appconfig['RAKEY']
rasecret = appconfig['RASECRET']
# no configuration file, the credentials should be retrieved with loutilities.apikey
else:
raprivuser = None
rakey = None
rasecret = None
# retrieve the membershipfile if not provided
membershipfile = _get_membershipfile(club, membershipfile=membershipfile, membercachefilename=membercachefilename, update=update, debug=debug, raprivuser=raprivuser, key=rakey, secret=rasecret)
# analyze the memberships
memberstats = analyzemembership(membershipfile)
# generate json file with membership statistics
membercount(memberstats, memberstatsfile)
# generate members csv file with member information
mapping = _get_summary_mapping()
members2file(membershipfile, mapping, membersummaryfile)
# for debugging
return membershipfile
def summarize(club, memberstatsfile, membersummaryfile, membershipfile=None, membercachefilename=None, update=False, debug=False, configfile=None):
#----------------------------------------------------------------------
'''
Summarize the membership stats and members for a given RunningAHEAD club.
If membershipfile is not supplied, retrieve the member data from RunningAHEAD
using the priviledged user token.
:param club: club slug for RunningAHEAD
:param memberstatsfile: output json file with member statistics
:param membersummaryfile: output csv file with summarized member information
:param membershipfile: filename, file handle, or list with member data (optional)
:param membercachefilename: name of optional file to cache detailed member data
:param update: True if cache should be updated
:param debug: True for requests debugging
:param configfile: optional configuration filename
'''
# configuration file supplied -- pull credentials from the app section
if configfile:
from loutilities.configparser import getitems
appconfig = getitems(configfile, 'app')
raprivuser = appconfig['RAPRIVUSER']
rakey = appconfig['RAKEY']
rasecret = appconfig['RASECRET']
# no configuration file, the credentials should be retrieved with loutilities.apikey
else:
raprivuser = None
rakey = None
rasecret = None
# retrieve the membershipfile if not provided
membershipfile = _get_membershipfile(club, membershipfile=membershipfile, membercachefilename=membercachefilename, update=update, debug=debug, raprivuser=raprivuser, key=rakey, secret=rasecret)
# analyze the memberships
memberstats = analyzemembership(membershipfile)
# generate json file with membership statistics
membercount(memberstats, memberstatsfile)
# generate members csv file with member information
mapping = _get_summary_mapping()
members2file(membershipfile, mapping, membersummaryfile)
# for debugging
return membershipfile
from __future__ import absolute_import
# standard
import os.path
# pypi
from celery import Celery
# homegrown
from loutilities.configparser import getitems
# note doe not config backend here else state does not come back
# see https://stackoverflow.com/questions/25495613/celery-getting-started-not-able-to-retrieve-results-always-pending
app = Celery('proj', include=['proj.tasks'])
# pull in configuration, configuration file in parent dir from this file
configpath = os.path.join(
os.path.sep.join(os.path.dirname(__file__).split(os.path.sep)[:-1]),
'celerytest.cfg')
config = getitems(configpath, 'celery')
# Optional configuration, see the application user guide.
app.config_from_object(config)
if __name__ == '__main__':
app.start()
def create_app(config_obj, configfiles=None, local_update=True):
'''
apply configuration object, then configuration filename
'''
global app
app = Flask('contracts')
app.config.from_object(config_obj)
if configfiles:
# backwards compatibility
if type(configfiles) == str:
configfiles = [configfiles]
for configfile in configfiles:
appconfig = getitems(configfile, 'app')
app.config.update(appconfig)
# tell jinja to remove linebreaks
app.jinja_env.trim_blocks = True
app.jinja_env.lstrip_blocks = True
# define product name (don't import nav until after app.jinja_env.globals['_productname'] set)
app.jinja_env.globals['_productname'] = app.config['THISAPP_PRODUCTNAME']
app.jinja_env.globals['_productname_text'] = app.config['THISAPP_PRODUCTNAME_TEXT']
# initialize database
from contracts.dbmodel import db
db.init_app(app)
# add loutilities tables-assets for js/css/template loading
# see https://adambard.com/blog/fresh-flask-setup/
# and https://webassets.readthedocs.io/en/latest/environment.html#webassets.env.Environment.load_path
# loutilities.__file__ is __init__.py file inside loutilities; os.path.split gets package directory
loutilitiespath = os.path.join(os.path.split(loutilities.__file__)[0], 'tables-assets', 'static')
@app.route('/loutilities/static/<path:filename>')
def loutilities_static(filename):
return send_from_directory(loutilitiespath, filename)
# bring in js, css assets here, because app needs to be created first
from .assets import asset_env, asset_bundles
with app.app_context():
# uncomment when working on #346
# # needs to be set before update_local_tables called and before UserSecurity() instantiated
# g.loutility = Application.query.filter_by(application=app.config['APP_LOUTILITY']).one()
#
# # update LocalUser and LocalInterest tables
# if local_update:
# update_local_tables()
# js/css files
asset_env.append_path(app.static_folder)
asset_env.append_path(loutilitiespath, '/loutilities/static')
# templates
loader = ChoiceLoader([
app.jinja_loader,
PackageLoader('loutilities', 'tables-assets/templates')
])
app.jinja_loader = loader
# initialize assets
asset_env.init_app(app)
asset_env.register(asset_bundles)
# Set up Flask-Security
from contracts.dbmodel import User, Role
global user_datastore, security
user_datastore = SQLAlchemyUserDatastore(db, User, Role)
security = Security(app, user_datastore)
# activate views
from contracts.views.frontend import bp as frontend
app.register_blueprint(frontend)
# need to force app context else get
# RuntimeError: Working outside of application context.
# RuntimeError: Attempted to generate a URL without the application context being pushed.
# see http://kronosapiens.github.io/blog/2014/08/14/understanding-contexts-in-flask.html
with app.app_context():
# admin views need to be defined within app context because of requests.addscripts() using url_for
from contracts.views.admin import bp as admin
app.register_blueprint(admin)
# import navigation after views created
from . import nav
# turn on logging
from .applogging import setlogging
setlogging()
# set up scoped session
from sqlalchemy.orm import scoped_session, sessionmaker
db.session = scoped_session(sessionmaker(autocommit=False,
autoflush=False,
bind=db.engine))
db.query = db.session.query_property()
# app back to caller
return app
from app import app
from loutilities.configparser import getitems
# tell jinja to remove linebreaks
app.jinja_env.trim_blocks = True
app.jinja_env.lstrip_blocks = True
# define product name (don't import nav until after app.jinja_env.globals['_rrdb_productname'] set)
# TODO: this really should be set in runningroutes.cfg
app.jinja_env.globals['_rrdb_productname'] = '<span class="brand-all"><span class="brand-left">route</span><span class="brand-right">tility</span></span>'
app.jinja_env.globals['_rrdb_productname_text'] = 'routetility'
# get configuration
configfile = os.environ['RR_CONFIG_FILE']
configpath = os.path.join(os.path.sep.join(os.path.dirname(__file__).split(os.path.sep)[:-1]), configfile)
appconfig = getitems(configpath, 'app')
app.config.update(appconfig)
# set static, templates if configured
app.static_folder = appconfig.get('STATIC_FOLDER', 'static')
app.template_folder = appconfig.get('TEMPLATE_FOLDER', 'templates')
# configure for debug
debug = app.config['DEBUG']
if debug:
app.config['SECRET_KEY'] = 'flask development key'
# nonview imports
import request
# import all views
def updatestravaclubactivitycache():
#----------------------------------------------------------------------
'''
script to update the strava club activity cache
usage: updatestravaclubactivitycache [-h] [-v] cachefile clubname
script to update the strava club activity cache
positional arguments:
cachefile pathname of file in which cache is saved
clubname full name of club as known to strava
optional arguments:
-h, --help show this help message and exit
-v, --version show program's version number and exit
'''
descr = '''
script to update the strava club activity cache
'''
parser = argparse.ArgumentParser(
description=descr,
formatter_class=argparse.RawDescriptionHelpFormatter,
version='{0} {1}'.format('running', version.__version__))
parser.add_argument('cachefile',
help="pathname of file in which cache is saved")
parser.add_argument('clubname',
help="full name of club as known to strava")
parser.add_argument('--configfile',
help='optional configuration filename',
default=None)
args = parser.parse_args()
# let user know what is going on
print 'Updating Strava club activity cache for "{}"'.format(args.clubname)
# configuration file supplied -- pull credentials from the app section
if args.configfile:
from loutilities.configparser import getitems
appconfig = getitems(args.configfile, 'app')
stravakey = appconfig['STRAVAKEY']
# no configuration file, the credentials should be retrieved with loutilities.apikey
else:
stravakey = None
# instantiate the Strava object, which opens the cache
ss = Strava(args.cachefile, key=stravakey)
# get the club id
clubs = ss.getathleteclubs()
clubid = None
for club in clubs:
if club['name'] == args.clubname:
clubid = club['id']
break
# error if we didn't find the club
if not clubid:
sys.exit('ERROR: club "{}" not found'.format(args.clubname))
# retrieve all the latest activities
activities = ss.getclubactivities(clubid)
numadded = ss.clubactivitycacheadded
cachesize = ss.clubactivitycachesize
# close the object, which saves the cache
ss.close()
# let user know how we did
print ' update complete:'
print ' {} activities received from Strava'.format(len(activities))
print ' added {} of these to cache'.format(numadded)
print ' new cache size = {}'.format(cachesize)
def create_app(config_obj, config_filename=None):
'''
apply configuration object, then configuration filename
'''
global app
app = Flask('runningroutes')
app.config.from_object(config_obj)
if config_filename:
appconfig = getitems(config_filename, 'app')
app.config.update(appconfig)
# tell jinja to remove linebreaks
app.jinja_env.trim_blocks = True
app.jinja_env.lstrip_blocks = True
# define product name (don't import nav until after app.jinja_env.globals['_productname'] set)
app.jinja_env.globals['_productname'] = app.config['THISAPP_PRODUCTNAME']
app.jinja_env.globals['_productname_text'] = app.config[
'THISAPP_PRODUCTNAME_TEXT']
# initialize database
from runningroutes.models import db
db.init_app(app)
# handle <interest> in URL - https://flask.palletsprojects.com/en/1.1.x/patterns/urlprocessors/
@app.url_value_preprocessor
def pull_interest(endpoint, values):
try:
g.interest = values.pop('interest', None)
except AttributeError:
g.interest = None
finally:
if not g.interest:
g.interest = request.args.get('interest', None)
# add loutilities tables-assets for js/css/template loading
# see https://adambard.com/blog/fresh-flask-setup/
# and https://webassets.readthedocs.io/en/latest/environment.html#webassets.env.Environment.load_path
# loutilities.__file__ is __init__.py file inside loutilities; os.path.split gets package directory
loutilitiespath = os.path.join(
os.path.split(loutilities.__file__)[0], 'tables-assets', 'static')
@app.route('/loutilities/static/<path:filename>')
def loutilities_static(filename):
return send_from_directory(loutilitiespath, filename)
# bring in js, css assets here, because app needs to be created first
from .assets import asset_env, asset_bundles
with app.app_context():
# js/css files
asset_env.append_path(app.static_folder)
asset_env.append_path(loutilitiespath, '/loutilities/static')
# templates
loader = ChoiceLoader([
app.jinja_loader,
PackageLoader('loutilities', 'tables-assets/templates')
])
app.jinja_loader = loader
# initialize assets
asset_env.init_app(app)
asset_env.register(asset_bundles)
# Set up Flask-Security
from runningroutes.models import User, Role
global user_datastore, security
user_datastore = SQLAlchemyUserDatastore(db, User, Role)
security = Security(app, user_datastore)
# Set up Flask-Mail [configuration in <application>.cfg
mail = Mail(app)
# activate views
from runningroutes.views.frontend import bp as frontend
app.register_blueprint(frontend)
from runningroutes.views.admin import bp as admin
app.register_blueprint(admin)
# need to force app context else get
# RuntimeError: Working outside of application context.
# RuntimeError: Attempted to generate a URL without the application context being pushed.
# see http://kronosapiens.github.io/blog/2014/08/14/understanding-contexts-in-flask.html
with app.app_context():
# import navigation after views created
from runningroutes import nav
# turn on logging
from .applogging import setlogging
setlogging()
# set up scoped session
from sqlalchemy.orm import scoped_session, sessionmaker
db.session = scoped_session(
sessionmaker(autocommit=False, autoflush=False, bind=db.engine))
db.query = db.session.query_property()
# handle favicon request for old browsers
app.add_url_rule('/favicon.ico',
endpoint='favicon',
redirect_to=url_for('static', filename='favicon.ico'))
# ----------------------------------------------------------------------
@app.before_request
def before_request():
if current_user.is_authenticated:
user = current_user
email = user.email
# used in layout.jinja2
app.jinja_env.globals['user_interests'] = sorted(
[{
'interest': i.interest,
'description': i.description
} for i in user.interests],
key=lambda a: a['description'].lower())
session['user_email'] = email
else:
# used in layout.jinja2
pubinterests = Interest.query.filter_by(public=True).all()
app.jinja_env.globals['user_interests'] = sorted(
[{
'interest': i.interest,
'description': i.description
} for i in pubinterests],
key=lambda a: a['description'].lower())
session.pop('user_email', None)
# ----------------------------------------------------------------------
@app.after_request
def after_request(response):
if not app.config['DEBUG']:
app.logger.info('{}: {} {} {}'.format(request.remote_addr,
request.method, request.url,
response.status_code))
return response
# app back to caller
return app
# create app and celery tasking back end
app = Flask('rrwebapp')
# define product name (don't import nav until after app.jinja_env.globals['_rrwebapp_productname'] set)
# TODO: this really should be set in rrwebapp.cfg
app.jinja_env.globals['_rrwebapp_productname'] = '<span class="brand-all"><span class="brand-left">score</span><span class="brand-right">tility</span></span>'
app.jinja_env.globals['_rrwebapp_productname_text'] = 'scoretility'
#from nav import productname
# tell jinja to remove linebreaks
app.jinja_env.trim_blocks = True
app.jinja_env.lstrip_blocks = True
# get configuration
configpath = os.path.join(os.path.sep.join(os.path.dirname(__file__).split(os.path.sep)[:-2]), 'rrwebapp.cfg')
appconfig = getitems(configpath, 'app')
app.config.update(appconfig)
celery = Celery('rrwebapp')
configpath = os.path.join(os.path.sep.join(os.path.dirname(__file__).split(os.path.sep)[:-2]), 'rrwebapp.cfg')
celeryconfig = getitems(configpath, 'celery')
celery.conf.update(celeryconfig)
import time
from loutilities import timeu
tu = timeu.asctime('%Y-%m-%d %H:%M:%S')
app.configtime = tu.epoch2asc(time.time())
# must set up logging after setting configuration
import applogging
class parameterError(Exception):
pass
# create app and get configuration
app = Flask(__name__)
dirname = os.path.dirname(__file__)
# one level up
dirname = os.path.dirname(dirname)
configdir = os.path.join(dirname, 'config')
configfile = "contracts.cfg"
configpath = os.path.join(configdir, configfile)
app.config.from_object(Production(configpath))
appconfig = getitems(configpath, 'app')
app.config.update(appconfig)
# set up database
db.init_app(app)
# set up scoped session
with app.app_context():
db.session = scoped_session(
sessionmaker(autocommit=False, autoflush=False, bind=db.engine))
db.query = db.session.query_property()
# turn on logging
setlogging()
# set up datatabase date formatter
def importmembers(configfile, debug=False, stats=False):
#----------------------------------------------------------------------
'''
import member data to mailchimp
configfile must have the following
[mailchimp]
RSU_CLUB: <runsignup club_id>
RSU_KEY: <key from runsignup partnership>
RSU_SECRET: <secret from runsignup partnership>
MC_KEY: <api key from MailChimp>
MC_LIST: <name of list of interest>
MC_GROUPNAMES: groupname1,groupname2,...
MC_SHADOWCATEGORY: <name of shadowcategory>
* shadowcategory groups are used to show desired inclusion but the group itself under other categories can be toggled by subscriber
* this is used to prevent the recipient from being added back to the group against their wishes
* if a groupname is not also under shadowcategory, it is only ticked if the subscriber was not present in the list prior to import
* this category's group names include all of the group names which are reserved for members
MC_CURRMEMBERGROUP: <name of group which is set for current members>
MC_PASTMEMBERGROUP: <name of group which is set for current and past members>
:param configfile: name of configuration file
:param debug: set to True for debug output
:param stats: set to True for stats output (INFO)
'''
# set up logging
thislogger.propagate = True
if debug:
# set up debug logging
thislogger.setLevel(logging.DEBUG)
elif stats:
# INFO logging
thislogger.setLevel(logging.INFO)
else:
# WARNING logging
thislogger.setLevel(logging.WARNING)
# load configuration
rsuconfig = getitems(configfile, 'runsignup')
mcconfig = getitems(configfile, 'mailchimp')
club_id = rsuconfig['RSU_CLUB']
rsukey = rsuconfig['RSU_KEY']
rsusecret = rsuconfig['RSU_SECRET']
mckey = mcconfig['MC_KEY']
mclist = mcconfig['MC_LIST']
mcgroupnames = mcconfig['MC_GROUPNAMES'].split(',')
mcshadowcategory = mcconfig['MC_SHADOWCATEGORY']
mcpastmembergroupname = mcconfig['MC_PASTMEMBERGROUP']
mccurrmembergroupname = mcconfig['MC_CURRMEMBERGROUP']
mctimeout = float(mcconfig['MC_TIMEOUT'])
# use Transform to simplify RunSignUp format
xform = Transform( {
'last' : lambda mem: mem['user']['last_name'],
'first' : lambda mem: mem['user']['first_name'],
'email' : lambda mem: mem['user']['email'] if 'email' in mem['user'] else '',
'primary' : lambda mem: mem['primary_member'] == 'T',
'start' : 'membership_start',
'end' : 'membership_end',
'modified' : 'last_modified',
},
# source and target are dicts, not objects
sourceattr=False,
targetattr=False
)
# download current member list from RunSignUp
# get current members from RunSignUp, transforming each to local format
# only save one member per email address, primary member preferred
rsu = RunSignUp(key=rsukey, secret=rsusecret, debug=debug)
rsu.open()
rsumembers = rsu.members(club_id)
rsucurrmembers = {}
for rsumember in rsumembers:
memberrec = {}
xform.transform(rsumember, memberrec)
memberkey = memberrec['email'].lower()
# only save if there's an email address
# the primary member takes precedence, but if different email for nonprimary members save those as well
if memberkey and (memberrec['primary'] or memberkey not in rsucurrmembers):
rsucurrmembers[memberkey] = memberrec
rsu.close()
# It's important not to add someone back to a group which they've decided not to receive
# emails from. For this reason, a membergroup is defined with the same group names as
# the real groups the user is interested in, for those groups which don't make up the whole
# list.
# download categories / groups from MailChimp
client = MailChimp(mc_api=mckey, timeout=mctimeout)
lists = client.lists.all(get_all=True, fields="lists.name,lists.id")
list_id = [lst['id'] for lst in lists['lists'] if lst['name'] == mclist][0]
categories = client.lists.interest_categories.all(list_id=list_id,fields="categories.title,categories.id")
# groups are for anyone, shadowgroups are for members only
groups = {}
shadowgroups = {}
# for debugging
allgroups = {}
for category in categories['categories']:
mcgroups = client.lists.interest_categories.interests.all(list_id=list_id,category_id=category['id'],fields="interests.name,interests.id")
for group in mcgroups['interests']:
# save for debug
allgroups[group['id']] = '{} / {}'.format(category['title'], group['name'])
# special group to track past members
if group['name'] == mcpastmembergroupname:
mcpastmembergroup = group['id']
# and current members
elif group['name'] == mccurrmembergroupname:
mccurrmembergroup = group['id']
# newly found members are enrolled in all groups
elif category['title'] != mcshadowcategory:
groups[group['name']] = group['id']
# shadowgroups is used to remember the state of member's only groups for previous members
# if a member's membership has expired they must be removed from any group(s) which have the same name as
# those within the shadowgroup(s) (associated groups)
# additionally if any nonmembers are found, they must be removed from the associated groups
# this last bit can happen if someone who is not a member tries to enroll in a members only group
else:
shadowgroups[group['name']] = group['id']
# set up specific groups for mc api
mcapi = Obj()
# members new to the list get all the groups
mcapi.newmember = { id : True for id in groups.values() + shadowgroups.values() + [mccurrmembergroup] + [mcpastmembergroup]}
# previous members who lapsed get the member groups disabled
mcapi.nonmember = { id : False for id in [groups[gname] for gname in groups.keys() if gname in shadowgroups] + [mccurrmembergroup] }
# members groups set to True, for mcapi.unsubscribed merge
mcapi.member = { id : True for id in shadowgroups.values() + [groups[gname] for gname in groups.keys() if gname in shadowgroups] + [mccurrmembergroup] + [mcpastmembergroup]}
# unsubscribed members who previously were not past members get member groups turned on and 'other' groups turned off
mcapi.unsubscribed = merge_dicts (mcapi.member, { id:False for id in [groups[gname] for gname in groups.keys() if gname not in shadowgroups] })
# retrieve all members of this mailchimp list
# key these into dict by id (md5 has of lower case email address)
tmpmcmembers = client.lists.members.all(list_id=list_id, get_all=True, fields='members.id,members.email_address,members.status,members.merge_fields,members.interests')
mcmembers = {}
for mcmember in tmpmcmembers['members']:
mcmembers[mcmember['id']] = mcmember
# collect some stats
stat = Stat(['addedtolist', 'newmemberunsubscribed', 'newmember', 'pastmember',
'nonmember', 'memberunsubscribedskipped', 'membercleanedskipped',
'mailchimperror'])
# loop through club members
# if club member is in mailchimp
# make sure shadowgroups are set (but don't change groups as these may have been adjusted by club member)
# don't change subscribed status
# pop off mcmembers as we want to deal with the leftovers later
# if club member is not already in mailchimp
# add assuming all groups (groups + shadowgroups)
for memberkey in rsucurrmembers:
clubmember = rsucurrmembers[memberkey]
mcmemberid = mcid(clubmember['email'])
thislogger.debug( 'processing {} {}'.format(clubmember['email'], mcmemberid) )
# if club member is in mailchimp
if mcmemberid in mcmembers:
mcmember = mcmembers.pop(mcmemberid)
# check if any changes are required
# change required if current member not set
if not mcmember['interests'][mccurrmembergroup]:
# if not past member, just set the needful
if not mcmember['interests'][mcpastmembergroup]:
# if subscribed, all groups are set
if mcmember['status'] == 'subscribed':
client.lists.members.update(list_id=list_id, subscriber_hash=mcmemberid, data={'interests' : mcapi.newmember})
stat.newmember += 1
# if unsubscribed, subscribe them to member stuff, but remove everything else
elif mcmember['status'] == 'unsubscribed':
try:
client.lists.members.update(list_id=list_id, subscriber_hash=mcmemberid, data={'interests' : mcapi.unsubscribed, 'status' : 'subscribed'})
stat.newmemberunsubscribed += 1
# MailChimp won't let us resubscribe this member
except MailChimpError as e:
thislogger.info('member unsubscribed, skipped: {}'.format(clubmember['email']))
stat.memberunsubscribedskipped += 1
# other statuses are skipped
else:
thislogger.info('member cleaned, skipped: {}'.format(clubmember['email']))
stat.membercleanedskipped += 1;
# past member, recall what they had set before for the member stuff
else:
pastmemberinterests = merge_dicts({ groups[gname] : mcmember['interests'][shadowgroups[gname]] for gname in shadowgroups.keys() },
{ mccurrmembergroup : True })
client.lists.members.update(list_id=list_id, subscriber_hash=mcmemberid, data={'interests' : pastmemberinterests})
stat.pastmember += 1
# if club member is missing from mailchimp
else:
try:
client.lists.members.create(list_id=list_id,
data={
'email_address' : clubmember['email'],
'merge_fields' : {'FNAME' : clubmember['first'], 'LNAME' : clubmember['last'] },
'interests' : mcapi.newmember,
'status' : 'subscribed'
})
stat.addedtolist += 1
except MailChimpError as e:
ed = e.args[0]
thislogger.warning('MailChimpError {} for {}: {}'.format(ed['title'], clubmember['email'], ed['detail']))
stat.mailchimperror += 1
# at this point, mcmembers have only those enrollees who are not in the club
# loop through each of these and make sure club only interests are removed
for mcmemberid in mcmembers:
mcmember = mcmembers[mcmemberid]
# change required if current member set
if mcmember['interests'][mccurrmembergroup]:
# save member interests for later if they rejoin
memberinterests = {shadowgroups[gname]:mcmember['interests'][groups[gname]] for gname in shadowgroups}
client.lists.members.update(list_id=list_id, subscriber_hash=mcmemberid, data={'interests' : merge_dicts(mcapi.nonmember, memberinterests)})
stat.nonmember += 1
# log stats
thislogger.info ( stat )
def create_app(config_obj, configfiles=None, init_for_operation=True):
'''
apply configuration object, then configuration files
'''
global app
app = Flask('members')
app.config.from_object(config_obj)
if configfiles:
# backwards compatibility
if type(configfiles) == str:
configfiles = [configfiles]
for configfile in configfiles:
appconfig = getitems(configfile, 'app')
app.config.update(appconfig)
# tell jinja to remove linebreaks
app.jinja_env.trim_blocks = True
app.jinja_env.lstrip_blocks = True
# define product name (don't import nav until after app.jinja_env.globals['_productname'] set)
app.jinja_env.globals['_productname'] = app.config['THISAPP_PRODUCTNAME']
app.jinja_env.globals['_productname_text'] = app.config['THISAPP_PRODUCTNAME_TEXT']
for configkey in ['SECURITY_EMAIL_SUBJECT_PASSWORD_RESET',
'SECURITY_EMAIL_SUBJECT_PASSWORD_CHANGE_NOTICE',
'SECURITY_EMAIL_SUBJECT_PASSWORD_NOTICE',
]:
app.config[configkey] = app.config[configkey].format(productname=app.config['THISAPP_PRODUCTNAME_TEXT'])
# initialize database
from .model import db
db.init_app(app)
# initialize uploads
if init_for_operation:
init_uploads(app)
# handle <interest> in URL - https://flask.palletsprojects.com/en/1.1.x/patterns/urlprocessors/
@app.url_value_preprocessor
def pull_interest(endpoint, values):
try:
g.interest = values.pop('interest', None)
except AttributeError:
g.interest = None
finally:
if not g.interest:
g.interest = request.args.get('interest', None)
# add loutilities tables-assets for js/css/template loading
# see https://adambard.com/blog/fresh-flask-setup/
# and https://webassets.readthedocs.io/en/latest/environment.html#webassets.env.Environment.load_path
# loutilities.__file__ is __init__.py file inside loutilities; os.path.split gets package directory
loutilitiespath = os.path.join(os.path.split(loutilities.__file__)[0], 'tables-assets', 'static')
@app.route('/loutilities/static/<path:filename>')
def loutilities_static(filename):
return send_from_directory(loutilitiespath, filename)
# bring in js, css assets here, because app needs to be created first
from .assets import asset_env, asset_bundles
with app.app_context():
# needs to be set before update_local_tables called and before UserSecurity() instantiated
g.loutility = Application.query.filter_by(application=app.config['APP_LOUTILITY']).one()
# update LocalUser and LocalInterest tables
if init_for_operation:
update_local_tables()
# js/css files
asset_env.append_path(app.static_folder)
asset_env.append_path(loutilitiespath, '/loutilities/static')
# templates
loader = ChoiceLoader([
app.jinja_loader,
PackageLoader('loutilities', 'tables-assets/templates')
])
app.jinja_loader = loader
# initialize assets
asset_env.init_app(app)
asset_env.register(asset_bundles)
# Set up Flask-Mail [configuration in <application>.cfg] and security mailer
mail = Mail(app)
def security_send_mail(subject, recipient, template, **context):
# this may be called from view which doesn't reference interest
# if so pick up user's first interest to get from_email address
if not g.interest:
g.interest = context['user'].interests[0].interest if context['user'].interests else None
if g.interest:
from_email = localinterest().from_email
# use default if user didn't have any interests
else:
from_email = current_app.config['SECURITY_EMAIL_SENDER']
# copied from flask_security.utils.send_mail
if isinstance(from_email, LocalProxy):
from_email = from_email._get_current_object()
ctx = ('security/email', template)
html = render_template('%s/%s.html' % ctx, **context)
text = render_template('%s/%s.txt' % ctx, **context)
sendmail(subject, from_email, recipient, html=html, text=text)
# Set up Flask-Security
global user_datastore, security
user_datastore = SQLAlchemyUserDatastore(db, User, Role)
security = UserSecurity(app, user_datastore, send_mail=security_send_mail)
# activate views
from .views import userrole as userroleviews
from loutilities.user.views import bp as userrole
app.register_blueprint(userrole)
from .views.frontend import bp as frontend
app.register_blueprint(frontend)
from .views.admin import bp as admin
app.register_blueprint(admin)
# need to force app context else get
# RuntimeError: Working outside of application context.
# RuntimeError: Attempted to generate a URL without the application context being pushed.
# see http://kronosapiens.github.io/blog/2014/08/14/understanding-contexts-in-flask.html
with app.app_context():
# import navigation after views created
from . import nav
# turn on logging
from .applogging import setlogging
setlogging()
# set up scoped session
from sqlalchemy.orm import scoped_session, sessionmaker
# see https://github.com/pallets/flask-sqlalchemy/blob/706982bb8a096220d29e5cef156950237753d89f/flask_sqlalchemy/__init__.py#L990
db.session = scoped_session(sessionmaker(autocommit=False,
autoflush=False,
binds=db.get_binds(app)))
db.query = db.session.query_property()
# handle favicon request for old browsers
app.add_url_rule('/favicon.ico', endpoint='favicon',
redirect_to=url_for('static', filename='favicon.ico'))
# ----------------------------------------------------------------------
@app.before_request
def before_request():
g.loutility = Application.query.filter_by(application=app.config['APP_LOUTILITY']).one()
if current_user.is_authenticated:
user = current_user
email = user.email
# used in layout.jinja2
app.jinja_env.globals['user_interests'] = sorted([{'interest': i.interest, 'description': i.description}
for i in user.interests if g.loutility in i.applications],
key=lambda a: a['description'].lower())
session['user_email'] = email
else:
# used in layout.jinja2
pubinterests = Interest.query.filter_by(public=True).all()
app.jinja_env.globals['user_interests'] = sorted([{'interest': i.interest, 'description': i.description}
for i in pubinterests if g.loutility in i.applications],
key=lambda a: a['description'].lower())
session.pop('user_email', None)
# ----------------------------------------------------------------------
@app.after_request
def after_request(response):
# # check if there are any changes needed to LocalUser table
# userupdated = User.query.order_by(desc('updated_at')).first().updated_at
# localuserupdated = LocalUser.query.order_by(desc('updated_at')).first().updated_at
# interestupdated = Interest.query.order_by(desc('updated_at')).first().updated_at
# localinterestupdated = LocalInterest.query.order_by(desc('updated_at')).first().updated_at
# if userupdated > localuserupdated or interestupdated > localinterestupdated:
# update_local_tables()
if not app.config['DEBUG']:
app.logger.info(
'{}: {} {} {}'.format(request.remote_addr, request.method, request.url, response.status_code))
return response
# app back to caller
return app
from __future__ import absolute_import
# standard
import os.path
# pypi
from celery import Celery
# homegrown
from loutilities.configparser import getitems
# note doe not config backend here else state does not come back
# see https://stackoverflow.com/questions/25495613/celery-getting-started-not-able-to-retrieve-results-always-pending
app = Celery('proj', include=['proj.tasks'])
# pull in configuration, configuration file in parent dir from this file
configpath = os.path.join(os.path.sep.join(os.path.dirname(__file__).split(os.path.sep)[:-1]), 'celerytest.cfg')
config = getitems(configpath, 'celery')
# Optional configuration, see the application user guide.
app.config_from_object(config)
if __name__ == '__main__':
app.start()
# standard
import os.path
# pypi
from flask import Flask
from celery import Celery
# homegrown
from loutilities.configparser import getitems
app = Flask(__name__)
# pull in app and celery configuration, configuration file in parent dir from this file
configpath = os.path.join(
os.path.sep.join(os.path.dirname(__file__).split(os.path.sep)[:-1]),
'celerytest.cfg')
appconfig = getitems(configpath, 'app')
app.config.update(appconfig)
celeryconfig = getitems(configpath, 'celery')
celery = Celery('proj')
celery.conf.update(celeryconfig)
# import views after configuration
import longtask
import debug
def updatestravaclubactivitycache():
#----------------------------------------------------------------------
'''
script to update the strava club activity cache
usage: updatestravaclubactivitycache [-h] [-v] cachefile clubname
script to update the strava club activity cache
positional arguments:
cachefile pathname of file in which cache is saved
clubname full name of club as known to strava
optional arguments:
-h, --help show this help message and exit
-v, --version show program's version number and exit
'''
descr = '''
script to update the strava club activity cache
'''
parser = argparse.ArgumentParser(description=descr,formatter_class=argparse.RawDescriptionHelpFormatter,
version='{0} {1}'.format('running',version.__version__))
parser.add_argument('cachefile', help="pathname of file in which cache is saved")
parser.add_argument('clubname', help="full name of club as known to strava")
parser.add_argument('--configfile', help='optional configuration filename', default=None)
args = parser.parse_args()
# let user know what is going on
print 'Updating Strava club activity cache for "{}"'.format(args.clubname)
# configuration file supplied -- pull credentials from the app section
if args.configfile:
from loutilities.configparser import getitems
appconfig = getitems(args.configfile, 'app')
stravakey = appconfig['STRAVAKEY']
# no configuration file, the credentials should be retrieved with loutilities.apikey
else:
stravakey = None
# instantiate the Strava object, which opens the cache
ss = Strava(args.cachefile, key=stravakey)
# get the club id
clubs = ss.getathleteclubs()
clubid = None
for club in clubs:
if club['name'] == args.clubname:
clubid = club['id']
break
# error if we didn't find the club
if not clubid:
sys.exit('ERROR: club "{}" not found'.format(args.clubname))
# retrieve all the latest activities
activities = ss.getclubactivities(clubid)
numadded = ss.clubactivitycacheadded
cachesize = ss.clubactivitycachesize
# close the object, which saves the cache
ss.close()
# let user know how we did
print ' update complete:'
print ' {} activities received from Strava'.format(len(activities))
print ' added {} of these to cache'.format(numadded)
print ' new cache size = {}'.format(cachesize)
def importmembers(configfile, debug=False, stats=False):
#----------------------------------------------------------------------
'''
import member data to mailchimp
configfile must have the following
[mailchimp]
RSU_CLUB: <runsignup club_id>
RSU_KEY: <key from runsignup partnership>
RSU_SECRET: <secret from runsignup partnership>
MC_KEY: <api key from MailChimp>
MC_LIST: <name of list of interest>
MC_GROUPNAMES: groupname1,groupname2,...
MC_SHADOWCATEGORY: <name of shadowcategory>
* shadowcategory groups are used to show desired inclusion but the group itself under other categories can be toggled by subscriber
* this is used to prevent the recipient from being added back to the group against their wishes
* if a groupname is not also under shadowcategory, it is only ticked if the subscriber was not present in the list prior to import
* this category's group names include all of the group names which are reserved for members
MC_CURRMEMBERGROUP: <name of group which is set for current members>
MC_PASTMEMBERGROUP: <name of group which is set for current and past members>
:param configfile: name of configuration file
:param debug: set to True for debug output
:param stats: set to True for stats output (INFO)
'''
# set up logging
thislogger.propagate = True
if debug:
# set up debug logging
thislogger.setLevel(logging.DEBUG)
elif stats:
# INFO logging
thislogger.setLevel(logging.INFO)
else:
# WARNING logging
thislogger.setLevel(logging.WARNING)
# load configuration
rsuconfig = getitems(configfile, 'runsignup')
mcconfig = getitems(configfile, 'mailchimp')
club_id = rsuconfig['RSU_CLUB']
rsukey = rsuconfig['RSU_KEY']
rsusecret = rsuconfig['RSU_SECRET']
mckey = mcconfig['MC_KEY']
mclist = mcconfig['MC_LIST']
mcgroupnames = mcconfig['MC_GROUPNAMES'].split(',')
mcshadowcategory = mcconfig['MC_SHADOWCATEGORY']
mcpastmembergroupname = mcconfig['MC_PASTMEMBERGROUP']
mccurrmembergroupname = mcconfig['MC_CURRMEMBERGROUP']
mctimeout = float(mcconfig['MC_TIMEOUT'])
# use Transform to simplify RunSignUp format
xform = Transform(
{
'last':
lambda mem: mem['user']['last_name'],
'first':
lambda mem: mem['user']['first_name'],
'email':
lambda mem: mem['user']['email'] if 'email' in mem['user'] else '',
'primary':
lambda mem: mem['primary_member'] == 'T',
'start':
'membership_start',
'end':
'membership_end',
'modified':
'last_modified',
},
# source and target are dicts, not objects
sourceattr=False,
targetattr=False)
# download current member list from RunSignUp
# get current members from RunSignUp, transforming each to local format
# only save one member per email address, primary member preferred
rsu = RunSignUp(key=rsukey, secret=rsusecret, debug=debug)
rsu.open()
rsumembers = rsu.members(club_id)
rsucurrmembers = {}
for rsumember in rsumembers:
memberrec = {}
xform.transform(rsumember, memberrec)
memberkey = memberrec['email'].lower()
# only save if there's an email address
# the primary member takes precedence, but if different email for nonprimary members save those as well
if memberkey and (memberrec['primary']
or memberkey not in rsucurrmembers):
rsucurrmembers[memberkey] = memberrec
rsu.close()
# It's important not to add someone back to a group which they've decided not to receive
# emails from. For this reason, a membergroup is defined with the same group names as
# the real groups the user is interested in, for those groups which don't make up the whole
# list.
# download categories / groups from MailChimp
client = MailChimp(mc_api=mckey, timeout=mctimeout)
lists = client.lists.all(get_all=True, fields="lists.name,lists.id")
list_id = [lst['id'] for lst in lists['lists'] if lst['name'] == mclist][0]
categories = client.lists.interest_categories.all(
list_id=list_id, fields="categories.title,categories.id")
# groups are for anyone, shadowgroups are for members only
groups = {}
shadowgroups = {}
# for debugging
allgroups = {}
for category in categories['categories']:
mcgroups = client.lists.interest_categories.interests.all(
list_id=list_id,
category_id=category['id'],
fields="interests.name,interests.id")
for group in mcgroups['interests']:
# save for debug
allgroups[group['id']] = '{} / {}'.format(category['title'],
group['name'])
# special group to track past members
if group['name'] == mcpastmembergroupname:
mcpastmembergroup = group['id']
# and current members
elif group['name'] == mccurrmembergroupname:
mccurrmembergroup = group['id']
# newly found members are enrolled in all groups
elif category['title'] != mcshadowcategory:
groups[group['name']] = group['id']
# shadowgroups is used to remember the state of member's only groups for previous members
# if a member's membership has expired they must be removed from any group(s) which have the same name as
# those within the shadowgroup(s) (associated groups)
# additionally if any nonmembers are found, they must be removed from the associated groups
# this last bit can happen if someone who is not a member tries to enroll in a members only group
else:
shadowgroups[group['name']] = group['id']
# set up specific groups for mc api
mcapi = Obj()
# members new to the list get all the groups
mcapi.newmember = {
id: True
for id in list(groups.values()) + list(shadowgroups.values()) +
[mccurrmembergroup] + [mcpastmembergroup]
}
# previous members who lapsed get the member groups disabled
mcapi.nonmember = {
id: False
for id in [
groups[gname]
for gname in list(groups.keys()) if gname in shadowgroups
] + [mccurrmembergroup]
}
# members groups set to True, for mcapi.unsubscribed merge
mcapi.member = {
id: True
for id in list(shadowgroups.values()) + [
groups[gname]
for gname in list(groups.keys()) if gname in shadowgroups
] + [mccurrmembergroup] + [mcpastmembergroup]
}
# unsubscribed members who previously were not past members get member groups turned on and 'other' groups turned off
mcapi.unsubscribed = merge_dicts(
mcapi.member, {
id: False
for id in [
groups[gname]
for gname in list(groups.keys()) if gname not in shadowgroups
]
})
# retrieve all members of this mailchimp list
# key these into dict by id (md5 has of lower case email address)
tmpmcmembers = client.lists.members.all(
list_id=list_id,
get_all=True,
fields=
'members.id,members.email_address,members.status,members.merge_fields,members.interests'
)
mcmembers = {}
for mcmember in tmpmcmembers['members']:
mcmembers[mcmember['id']] = mcmember
# collect some stats
stat = Stat([
'addedtolist', 'newmemberunsubscribed', 'newmember', 'pastmember',
'nonmember', 'memberunsubscribedskipped', 'membercleanedskipped',
'mailchimperror'
])
# loop through club members
# if club member is in mailchimp
# make sure shadowgroups are set (but don't change groups as these may have been adjusted by club member)
# don't change subscribed status
# pop off mcmembers as we want to deal with the leftovers later
# if club member is not already in mailchimp
# add assuming all groups (groups + shadowgroups)
for memberkey in rsucurrmembers:
clubmember = rsucurrmembers[memberkey]
mcmemberid = mcid(clubmember['email'])
thislogger.debug('processing {} {}'.format(clubmember['email'],
mcmemberid))
# if club member is in mailchimp
if mcmemberid in mcmembers:
mcmember = mcmembers.pop(mcmemberid)
# check if any changes are required
# change required if current member not set
if not mcmember['interests'][mccurrmembergroup]:
# if not past member, just set the needful
if not mcmember['interests'][mcpastmembergroup]:
# if subscribed, all groups are set
if mcmember['status'] == 'subscribed':
client.lists.members.update(
list_id=list_id,
subscriber_hash=mcmemberid,
data={'interests': mcapi.newmember})
stat.newmember += 1
# if unsubscribed, subscribe them to member stuff, but remove everything else
elif mcmember['status'] == 'unsubscribed':
try:
client.lists.members.update(
list_id=list_id,
subscriber_hash=mcmemberid,
data={
'interests': mcapi.unsubscribed,
'status': 'subscribed'
})
stat.newmemberunsubscribed += 1
# MailChimp won't let us resubscribe this member
except MailChimpError as e:
thislogger.info(
'member unsubscribed, skipped: {}'.format(
clubmember['email']))
stat.memberunsubscribedskipped += 1
# other statuses are skipped
else:
thislogger.info('member cleaned, skipped: {}'.format(
clubmember['email']))
stat.membercleanedskipped += 1
# past member, recall what they had set before for the member stuff
else:
pastmemberinterests = merge_dicts(
{
groups[gname]:
mcmember['interests'][shadowgroups[gname]]
for gname in list(shadowgroups.keys())
}, {mccurrmembergroup: True})
client.lists.members.update(
list_id=list_id,
subscriber_hash=mcmemberid,
data={'interests': pastmemberinterests})
stat.pastmember += 1
# if club member is missing from mailchimp
else:
try:
client.lists.members.create(list_id=list_id,
data={
'email_address':
clubmember['email'],
'merge_fields': {
'FNAME':
clubmember['first'],
'LNAME': clubmember['last']
},
'interests':
mcapi.newmember,
'status':
'subscribed'
})
stat.addedtolist += 1
except MailChimpError as e:
ed = e.args[0]
thislogger.warning('MailChimpError {} for {}: {}'.format(
ed['title'], clubmember['email'], ed['detail']))
stat.mailchimperror += 1
# at this point, mcmembers have only those enrollees who are not in the club
# loop through each of these and make sure club only interests are removed
for mcmemberid in mcmembers:
mcmember = mcmembers[mcmemberid]
# change required if current member set
if mcmember['interests'][mccurrmembergroup]:
# save member interests for later if they rejoin
memberinterests = {
shadowgroups[gname]: mcmember['interests'][groups[gname]]
for gname in shadowgroups
}
client.lists.members.update(list_id=list_id,
subscriber_hash=mcmemberid,
data={
'interests':
merge_dicts(
mcapi.nonmember,
memberinterests)
})
stat.nonmember += 1
# log stats
thislogger.info(stat)
