def getconfig(configfile): #---------------------------------------------------------------------- ''' get configuratoin from configfile :param configfile: file containing configuration, formatted per python ConfigParser :rtype: dict with configuration key, value pairs ''' # combine app and runsignup sections config = getitems(configfile, 'app') config.update(getitems(configfile, 'runsignup')) return config
def summarize(configfile, debug=False): #---------------------------------------------------------------------- ''' Summarize the membership stats and members for a given runsignup club. :param configfile: configuration filename :param debug: True for requests debugging ''' # configuration file supplied -- pull credentials from the app section appconfig = getitems(configfile, 'runsignup') club = appconfig['RSU_CLUB'] membercachefile = appconfig['RSU_CACHEFILE'] memberstatsfile = appconfig['RSU_STATSFILE'] key = appconfig['RSU_KEY'] secret = appconfig['RSU_SECRET'] # update member cache file # note this update is done under lock to prevent any apache thread from disrupting it members = updatemembercache(club, membercachefile, key=key, secret=secret, debug=debug) # analyze the memberships memberstats = analyzemembership(membercachefile, statsfile=memberstatsfile) # for debugging return members, memberstats
def __init__(self, configfiles): if type(configfiles) == str: configfiles = [configfiles] # connect to database based on configuration config = {} for configfile in configfiles: config.update(getitems(configfile, 'database')) dbuser = config['dbuser'] password = config['dbpassword'] dbserver = config['dbserver'] dbname = config['dbname'] # app.logger.debug('using mysql://{uname}:*******@{server}/{dbname}'.format(uname=dbuser,server=dbserver,dbname=dbname)) db_uri = 'mysql://{uname}:{pw}@{server}/{dbname}'.format( uname=dbuser, pw=password, server=dbserver, dbname=dbname) self.SQLALCHEMY_DATABASE_URI = db_uri # https://flask-sqlalchemy.palletsprojects.com/en/2.x/binds/ userdbuser = config['userdbuser'] userpassword = config['userdbpassword'] userdbserver = config['userdbserver'] userdbname = config['userdbname'] userdb_uri = 'mysql://{uname}:{pw}@{server}/{dbname}'.format( uname=userdbuser, pw=userpassword, server=userdbserver, dbname=userdbname) self.SQLALCHEMY_BINDS = {'users': userdb_uri}
def __init__(self, configpath): # connect to database based on configuration config = getitems(configpath, 'database') dbuser = config['dbuser'] password = config['dbpassword'] dbserver = config['dbserver'] dbname = config['dbname'] # app.logger.debug('using mysql://{uname}:*******@{server}/{dbname}'.format(uname=dbuser,server=dbserver,dbname=dbname)) db_uri = 'mysql://{uname}:{pw}@{server}/{dbname}'.format( uname=dbuser, pw=password, server=dbserver, dbname=dbname) self.SQLALCHEMY_DATABASE_URI = db_uri
def __init__(self, configfiles): if type(configfiles) == str: configfiles = [configfiles] # connect to database based on configuration config = {} for configfile in configfiles: config.update(getitems(configfile, 'database')) dbuser = config['dbuser'] password = config['dbpassword'] dbserver = config['dbserver'] dbname = config['dbname'] db_uri = 'mysql://{uname}:{pw}@{server}/{dbname}'.format( uname=dbuser, pw=password, server=dbserver, dbname=dbname) self.SQLALCHEMY_DATABASE_URI = db_uri
def summarize(club, memberstatsfile, membersummaryfile, membershipfile=None, membercachefilename=None, update=False, debug=False, configfile=None): #---------------------------------------------------------------------- ''' Summarize the membership stats and members for a given RunningAHEAD club. If membershipfile is not supplied, retrieve the member data from RunningAHEAD using the priviledged user token. :param club: club slug for RunningAHEAD :param memberstatsfile: output json file with member statistics :param membersummaryfile: output csv file with summarized member information :param membershipfile: filename, file handle, or list with member data (optional) :param membercachefilename: name of optional file to cache detailed member data :param update: True if cache should be updated :param debug: True for requests debugging :param configfile: optional configuration filename ''' # configuration file supplied -- pull credentials from the app section if configfile: from loutilities.configparser import getitems appconfig = getitems(configfile, 'app') raprivuser = appconfig['RAPRIVUSER'] rakey = appconfig['RAKEY'] rasecret = appconfig['RASECRET'] # no configuration file, the credentials should be retrieved with loutilities.apikey else: raprivuser = None rakey = None rasecret = None # retrieve the membershipfile if not provided membershipfile = _get_membershipfile(club, membershipfile=membershipfile, membercachefilename=membercachefilename, update=update, debug=debug, raprivuser=raprivuser, key=rakey, secret=rasecret) # analyze the memberships memberstats = analyzemembership(membershipfile) # generate json file with membership statistics membercount(memberstats, memberstatsfile) # generate members csv file with member information mapping = _get_summary_mapping() members2file(membershipfile, mapping, membersummaryfile) # for debugging return membershipfile
from __future__ import absolute_import # standard import os.path # pypi from celery import Celery # homegrown from loutilities.configparser import getitems # note doe not config backend here else state does not come back # see https://stackoverflow.com/questions/25495613/celery-getting-started-not-able-to-retrieve-results-always-pending app = Celery('proj', include=['proj.tasks']) # pull in configuration, configuration file in parent dir from this file configpath = os.path.join( os.path.sep.join(os.path.dirname(__file__).split(os.path.sep)[:-1]), 'celerytest.cfg') config = getitems(configpath, 'celery') # Optional configuration, see the application user guide. app.config_from_object(config) if __name__ == '__main__': app.start()
def create_app(config_obj, configfiles=None, local_update=True): ''' apply configuration object, then configuration filename ''' global app app = Flask('contracts') app.config.from_object(config_obj) if configfiles: # backwards compatibility if type(configfiles) == str: configfiles = [configfiles] for configfile in configfiles: appconfig = getitems(configfile, 'app') app.config.update(appconfig) # tell jinja to remove linebreaks app.jinja_env.trim_blocks = True app.jinja_env.lstrip_blocks = True # define product name (don't import nav until after app.jinja_env.globals['_productname'] set) app.jinja_env.globals['_productname'] = app.config['THISAPP_PRODUCTNAME'] app.jinja_env.globals['_productname_text'] = app.config['THISAPP_PRODUCTNAME_TEXT'] # initialize database from contracts.dbmodel import db db.init_app(app) # add loutilities tables-assets for js/css/template loading # see https://adambard.com/blog/fresh-flask-setup/ # and https://webassets.readthedocs.io/en/latest/environment.html#webassets.env.Environment.load_path # loutilities.__file__ is __init__.py file inside loutilities; os.path.split gets package directory loutilitiespath = os.path.join(os.path.split(loutilities.__file__)[0], 'tables-assets', 'static') @app.route('/loutilities/static/<path:filename>') def loutilities_static(filename): return send_from_directory(loutilitiespath, filename) # bring in js, css assets here, because app needs to be created first from .assets import asset_env, asset_bundles with app.app_context(): # uncomment when working on #346 # # needs to be set before update_local_tables called and before UserSecurity() instantiated # g.loutility = Application.query.filter_by(application=app.config['APP_LOUTILITY']).one() # # # update LocalUser and LocalInterest tables # if local_update: # update_local_tables() # js/css files asset_env.append_path(app.static_folder) asset_env.append_path(loutilitiespath, '/loutilities/static') # templates loader = ChoiceLoader([ app.jinja_loader, PackageLoader('loutilities', 'tables-assets/templates') ]) app.jinja_loader = loader # initialize assets asset_env.init_app(app) asset_env.register(asset_bundles) # Set up Flask-Security from contracts.dbmodel import User, Role global user_datastore, security user_datastore = SQLAlchemyUserDatastore(db, User, Role) security = Security(app, user_datastore) # activate views from contracts.views.frontend import bp as frontend app.register_blueprint(frontend) # need to force app context else get # RuntimeError: Working outside of application context. # RuntimeError: Attempted to generate a URL without the application context being pushed. # see http://kronosapiens.github.io/blog/2014/08/14/understanding-contexts-in-flask.html with app.app_context(): # admin views need to be defined within app context because of requests.addscripts() using url_for from contracts.views.admin import bp as admin app.register_blueprint(admin) # import navigation after views created from . import nav # turn on logging from .applogging import setlogging setlogging() # set up scoped session from sqlalchemy.orm import scoped_session, sessionmaker db.session = scoped_session(sessionmaker(autocommit=False, autoflush=False, bind=db.engine)) db.query = db.session.query_property() # app back to caller return app
from app import app from loutilities.configparser import getitems # tell jinja to remove linebreaks app.jinja_env.trim_blocks = True app.jinja_env.lstrip_blocks = True # define product name (don't import nav until after app.jinja_env.globals['_rrdb_productname'] set) # TODO: this really should be set in runningroutes.cfg app.jinja_env.globals['_rrdb_productname'] = '<span class="brand-all"><span class="brand-left">route</span><span class="brand-right">tility</span></span>' app.jinja_env.globals['_rrdb_productname_text'] = 'routetility' # get configuration configfile = os.environ['RR_CONFIG_FILE'] configpath = os.path.join(os.path.sep.join(os.path.dirname(__file__).split(os.path.sep)[:-1]), configfile) appconfig = getitems(configpath, 'app') app.config.update(appconfig) # set static, templates if configured app.static_folder = appconfig.get('STATIC_FOLDER', 'static') app.template_folder = appconfig.get('TEMPLATE_FOLDER', 'templates') # configure for debug debug = app.config['DEBUG'] if debug: app.config['SECRET_KEY'] = 'flask development key' # nonview imports import request # import all views
def updatestravaclubactivitycache(): #---------------------------------------------------------------------- ''' script to update the strava club activity cache usage: updatestravaclubactivitycache [-h] [-v] cachefile clubname script to update the strava club activity cache positional arguments: cachefile pathname of file in which cache is saved clubname full name of club as known to strava optional arguments: -h, --help show this help message and exit -v, --version show program's version number and exit ''' descr = ''' script to update the strava club activity cache ''' parser = argparse.ArgumentParser( description=descr, formatter_class=argparse.RawDescriptionHelpFormatter, version='{0} {1}'.format('running', version.__version__)) parser.add_argument('cachefile', help="pathname of file in which cache is saved") parser.add_argument('clubname', help="full name of club as known to strava") parser.add_argument('--configfile', help='optional configuration filename', default=None) args = parser.parse_args() # let user know what is going on print 'Updating Strava club activity cache for "{}"'.format(args.clubname) # configuration file supplied -- pull credentials from the app section if args.configfile: from loutilities.configparser import getitems appconfig = getitems(args.configfile, 'app') stravakey = appconfig['STRAVAKEY'] # no configuration file, the credentials should be retrieved with loutilities.apikey else: stravakey = None # instantiate the Strava object, which opens the cache ss = Strava(args.cachefile, key=stravakey) # get the club id clubs = ss.getathleteclubs() clubid = None for club in clubs: if club['name'] == args.clubname: clubid = club['id'] break # error if we didn't find the club if not clubid: sys.exit('ERROR: club "{}" not found'.format(args.clubname)) # retrieve all the latest activities activities = ss.getclubactivities(clubid) numadded = ss.clubactivitycacheadded cachesize = ss.clubactivitycachesize # close the object, which saves the cache ss.close() # let user know how we did print ' update complete:' print ' {} activities received from Strava'.format(len(activities)) print ' added {} of these to cache'.format(numadded) print ' new cache size = {}'.format(cachesize)
def create_app(config_obj, config_filename=None): ''' apply configuration object, then configuration filename ''' global app app = Flask('runningroutes') app.config.from_object(config_obj) if config_filename: appconfig = getitems(config_filename, 'app') app.config.update(appconfig) # tell jinja to remove linebreaks app.jinja_env.trim_blocks = True app.jinja_env.lstrip_blocks = True # define product name (don't import nav until after app.jinja_env.globals['_productname'] set) app.jinja_env.globals['_productname'] = app.config['THISAPP_PRODUCTNAME'] app.jinja_env.globals['_productname_text'] = app.config[ 'THISAPP_PRODUCTNAME_TEXT'] # initialize database from runningroutes.models import db db.init_app(app) # handle <interest> in URL - https://flask.palletsprojects.com/en/1.1.x/patterns/urlprocessors/ @app.url_value_preprocessor def pull_interest(endpoint, values): try: g.interest = values.pop('interest', None) except AttributeError: g.interest = None finally: if not g.interest: g.interest = request.args.get('interest', None) # add loutilities tables-assets for js/css/template loading # see https://adambard.com/blog/fresh-flask-setup/ # and https://webassets.readthedocs.io/en/latest/environment.html#webassets.env.Environment.load_path # loutilities.__file__ is __init__.py file inside loutilities; os.path.split gets package directory loutilitiespath = os.path.join( os.path.split(loutilities.__file__)[0], 'tables-assets', 'static') @app.route('/loutilities/static/<path:filename>') def loutilities_static(filename): return send_from_directory(loutilitiespath, filename) # bring in js, css assets here, because app needs to be created first from .assets import asset_env, asset_bundles with app.app_context(): # js/css files asset_env.append_path(app.static_folder) asset_env.append_path(loutilitiespath, '/loutilities/static') # templates loader = ChoiceLoader([ app.jinja_loader, PackageLoader('loutilities', 'tables-assets/templates') ]) app.jinja_loader = loader # initialize assets asset_env.init_app(app) asset_env.register(asset_bundles) # Set up Flask-Security from runningroutes.models import User, Role global user_datastore, security user_datastore = SQLAlchemyUserDatastore(db, User, Role) security = Security(app, user_datastore) # Set up Flask-Mail [configuration in <application>.cfg mail = Mail(app) # activate views from runningroutes.views.frontend import bp as frontend app.register_blueprint(frontend) from runningroutes.views.admin import bp as admin app.register_blueprint(admin) # need to force app context else get # RuntimeError: Working outside of application context. # RuntimeError: Attempted to generate a URL without the application context being pushed. # see http://kronosapiens.github.io/blog/2014/08/14/understanding-contexts-in-flask.html with app.app_context(): # import navigation after views created from runningroutes import nav # turn on logging from .applogging import setlogging setlogging() # set up scoped session from sqlalchemy.orm import scoped_session, sessionmaker db.session = scoped_session( sessionmaker(autocommit=False, autoflush=False, bind=db.engine)) db.query = db.session.query_property() # handle favicon request for old browsers app.add_url_rule('/favicon.ico', endpoint='favicon', redirect_to=url_for('static', filename='favicon.ico')) # ---------------------------------------------------------------------- @app.before_request def before_request(): if current_user.is_authenticated: user = current_user email = user.email # used in layout.jinja2 app.jinja_env.globals['user_interests'] = sorted( [{ 'interest': i.interest, 'description': i.description } for i in user.interests], key=lambda a: a['description'].lower()) session['user_email'] = email else: # used in layout.jinja2 pubinterests = Interest.query.filter_by(public=True).all() app.jinja_env.globals['user_interests'] = sorted( [{ 'interest': i.interest, 'description': i.description } for i in pubinterests], key=lambda a: a['description'].lower()) session.pop('user_email', None) # ---------------------------------------------------------------------- @app.after_request def after_request(response): if not app.config['DEBUG']: app.logger.info('{}: {} {} {}'.format(request.remote_addr, request.method, request.url, response.status_code)) return response # app back to caller return app
# create app and celery tasking back end app = Flask('rrwebapp') # define product name (don't import nav until after app.jinja_env.globals['_rrwebapp_productname'] set) # TODO: this really should be set in rrwebapp.cfg app.jinja_env.globals['_rrwebapp_productname'] = '<span class="brand-all"><span class="brand-left">score</span><span class="brand-right">tility</span></span>' app.jinja_env.globals['_rrwebapp_productname_text'] = 'scoretility' #from nav import productname # tell jinja to remove linebreaks app.jinja_env.trim_blocks = True app.jinja_env.lstrip_blocks = True # get configuration configpath = os.path.join(os.path.sep.join(os.path.dirname(__file__).split(os.path.sep)[:-2]), 'rrwebapp.cfg') appconfig = getitems(configpath, 'app') app.config.update(appconfig) celery = Celery('rrwebapp') configpath = os.path.join(os.path.sep.join(os.path.dirname(__file__).split(os.path.sep)[:-2]), 'rrwebapp.cfg') celeryconfig = getitems(configpath, 'celery') celery.conf.update(celeryconfig) import time from loutilities import timeu tu = timeu.asctime('%Y-%m-%d %H:%M:%S') app.configtime = tu.epoch2asc(time.time()) # must set up logging after setting configuration import applogging
class parameterError(Exception): pass # create app and get configuration app = Flask(__name__) dirname = os.path.dirname(__file__) # one level up dirname = os.path.dirname(dirname) configdir = os.path.join(dirname, 'config') configfile = "contracts.cfg" configpath = os.path.join(configdir, configfile) app.config.from_object(Production(configpath)) appconfig = getitems(configpath, 'app') app.config.update(appconfig) # set up database db.init_app(app) # set up scoped session with app.app_context(): db.session = scoped_session( sessionmaker(autocommit=False, autoflush=False, bind=db.engine)) db.query = db.session.query_property() # turn on logging setlogging() # set up datatabase date formatter
def importmembers(configfile, debug=False, stats=False): #---------------------------------------------------------------------- ''' import member data to mailchimp configfile must have the following [mailchimp] RSU_CLUB: <runsignup club_id> RSU_KEY: <key from runsignup partnership> RSU_SECRET: <secret from runsignup partnership> MC_KEY: <api key from MailChimp> MC_LIST: <name of list of interest> MC_GROUPNAMES: groupname1,groupname2,... MC_SHADOWCATEGORY: <name of shadowcategory> * shadowcategory groups are used to show desired inclusion but the group itself under other categories can be toggled by subscriber * this is used to prevent the recipient from being added back to the group against their wishes * if a groupname is not also under shadowcategory, it is only ticked if the subscriber was not present in the list prior to import * this category's group names include all of the group names which are reserved for members MC_CURRMEMBERGROUP: <name of group which is set for current members> MC_PASTMEMBERGROUP: <name of group which is set for current and past members> :param configfile: name of configuration file :param debug: set to True for debug output :param stats: set to True for stats output (INFO) ''' # set up logging thislogger.propagate = True if debug: # set up debug logging thislogger.setLevel(logging.DEBUG) elif stats: # INFO logging thislogger.setLevel(logging.INFO) else: # WARNING logging thislogger.setLevel(logging.WARNING) # load configuration rsuconfig = getitems(configfile, 'runsignup') mcconfig = getitems(configfile, 'mailchimp') club_id = rsuconfig['RSU_CLUB'] rsukey = rsuconfig['RSU_KEY'] rsusecret = rsuconfig['RSU_SECRET'] mckey = mcconfig['MC_KEY'] mclist = mcconfig['MC_LIST'] mcgroupnames = mcconfig['MC_GROUPNAMES'].split(',') mcshadowcategory = mcconfig['MC_SHADOWCATEGORY'] mcpastmembergroupname = mcconfig['MC_PASTMEMBERGROUP'] mccurrmembergroupname = mcconfig['MC_CURRMEMBERGROUP'] mctimeout = float(mcconfig['MC_TIMEOUT']) # use Transform to simplify RunSignUp format xform = Transform( { 'last' : lambda mem: mem['user']['last_name'], 'first' : lambda mem: mem['user']['first_name'], 'email' : lambda mem: mem['user']['email'] if 'email' in mem['user'] else '', 'primary' : lambda mem: mem['primary_member'] == 'T', 'start' : 'membership_start', 'end' : 'membership_end', 'modified' : 'last_modified', }, # source and target are dicts, not objects sourceattr=False, targetattr=False ) # download current member list from RunSignUp # get current members from RunSignUp, transforming each to local format # only save one member per email address, primary member preferred rsu = RunSignUp(key=rsukey, secret=rsusecret, debug=debug) rsu.open() rsumembers = rsu.members(club_id) rsucurrmembers = {} for rsumember in rsumembers: memberrec = {} xform.transform(rsumember, memberrec) memberkey = memberrec['email'].lower() # only save if there's an email address # the primary member takes precedence, but if different email for nonprimary members save those as well if memberkey and (memberrec['primary'] or memberkey not in rsucurrmembers): rsucurrmembers[memberkey] = memberrec rsu.close() # It's important not to add someone back to a group which they've decided not to receive # emails from. For this reason, a membergroup is defined with the same group names as # the real groups the user is interested in, for those groups which don't make up the whole # list. # download categories / groups from MailChimp client = MailChimp(mc_api=mckey, timeout=mctimeout) lists = client.lists.all(get_all=True, fields="lists.name,lists.id") list_id = [lst['id'] for lst in lists['lists'] if lst['name'] == mclist][0] categories = client.lists.interest_categories.all(list_id=list_id,fields="categories.title,categories.id") # groups are for anyone, shadowgroups are for members only groups = {} shadowgroups = {} # for debugging allgroups = {} for category in categories['categories']: mcgroups = client.lists.interest_categories.interests.all(list_id=list_id,category_id=category['id'],fields="interests.name,interests.id") for group in mcgroups['interests']: # save for debug allgroups[group['id']] = '{} / {}'.format(category['title'], group['name']) # special group to track past members if group['name'] == mcpastmembergroupname: mcpastmembergroup = group['id'] # and current members elif group['name'] == mccurrmembergroupname: mccurrmembergroup = group['id'] # newly found members are enrolled in all groups elif category['title'] != mcshadowcategory: groups[group['name']] = group['id'] # shadowgroups is used to remember the state of member's only groups for previous members # if a member's membership has expired they must be removed from any group(s) which have the same name as # those within the shadowgroup(s) (associated groups) # additionally if any nonmembers are found, they must be removed from the associated groups # this last bit can happen if someone who is not a member tries to enroll in a members only group else: shadowgroups[group['name']] = group['id'] # set up specific groups for mc api mcapi = Obj() # members new to the list get all the groups mcapi.newmember = { id : True for id in groups.values() + shadowgroups.values() + [mccurrmembergroup] + [mcpastmembergroup]} # previous members who lapsed get the member groups disabled mcapi.nonmember = { id : False for id in [groups[gname] for gname in groups.keys() if gname in shadowgroups] + [mccurrmembergroup] } # members groups set to True, for mcapi.unsubscribed merge mcapi.member = { id : True for id in shadowgroups.values() + [groups[gname] for gname in groups.keys() if gname in shadowgroups] + [mccurrmembergroup] + [mcpastmembergroup]} # unsubscribed members who previously were not past members get member groups turned on and 'other' groups turned off mcapi.unsubscribed = merge_dicts (mcapi.member, { id:False for id in [groups[gname] for gname in groups.keys() if gname not in shadowgroups] }) # retrieve all members of this mailchimp list # key these into dict by id (md5 has of lower case email address) tmpmcmembers = client.lists.members.all(list_id=list_id, get_all=True, fields='members.id,members.email_address,members.status,members.merge_fields,members.interests') mcmembers = {} for mcmember in tmpmcmembers['members']: mcmembers[mcmember['id']] = mcmember # collect some stats stat = Stat(['addedtolist', 'newmemberunsubscribed', 'newmember', 'pastmember', 'nonmember', 'memberunsubscribedskipped', 'membercleanedskipped', 'mailchimperror']) # loop through club members # if club member is in mailchimp # make sure shadowgroups are set (but don't change groups as these may have been adjusted by club member) # don't change subscribed status # pop off mcmembers as we want to deal with the leftovers later # if club member is not already in mailchimp # add assuming all groups (groups + shadowgroups) for memberkey in rsucurrmembers: clubmember = rsucurrmembers[memberkey] mcmemberid = mcid(clubmember['email']) thislogger.debug( 'processing {} {}'.format(clubmember['email'], mcmemberid) ) # if club member is in mailchimp if mcmemberid in mcmembers: mcmember = mcmembers.pop(mcmemberid) # check if any changes are required # change required if current member not set if not mcmember['interests'][mccurrmembergroup]: # if not past member, just set the needful if not mcmember['interests'][mcpastmembergroup]: # if subscribed, all groups are set if mcmember['status'] == 'subscribed': client.lists.members.update(list_id=list_id, subscriber_hash=mcmemberid, data={'interests' : mcapi.newmember}) stat.newmember += 1 # if unsubscribed, subscribe them to member stuff, but remove everything else elif mcmember['status'] == 'unsubscribed': try: client.lists.members.update(list_id=list_id, subscriber_hash=mcmemberid, data={'interests' : mcapi.unsubscribed, 'status' : 'subscribed'}) stat.newmemberunsubscribed += 1 # MailChimp won't let us resubscribe this member except MailChimpError as e: thislogger.info('member unsubscribed, skipped: {}'.format(clubmember['email'])) stat.memberunsubscribedskipped += 1 # other statuses are skipped else: thislogger.info('member cleaned, skipped: {}'.format(clubmember['email'])) stat.membercleanedskipped += 1; # past member, recall what they had set before for the member stuff else: pastmemberinterests = merge_dicts({ groups[gname] : mcmember['interests'][shadowgroups[gname]] for gname in shadowgroups.keys() }, { mccurrmembergroup : True }) client.lists.members.update(list_id=list_id, subscriber_hash=mcmemberid, data={'interests' : pastmemberinterests}) stat.pastmember += 1 # if club member is missing from mailchimp else: try: client.lists.members.create(list_id=list_id, data={ 'email_address' : clubmember['email'], 'merge_fields' : {'FNAME' : clubmember['first'], 'LNAME' : clubmember['last'] }, 'interests' : mcapi.newmember, 'status' : 'subscribed' }) stat.addedtolist += 1 except MailChimpError as e: ed = e.args[0] thislogger.warning('MailChimpError {} for {}: {}'.format(ed['title'], clubmember['email'], ed['detail'])) stat.mailchimperror += 1 # at this point, mcmembers have only those enrollees who are not in the club # loop through each of these and make sure club only interests are removed for mcmemberid in mcmembers: mcmember = mcmembers[mcmemberid] # change required if current member set if mcmember['interests'][mccurrmembergroup]: # save member interests for later if they rejoin memberinterests = {shadowgroups[gname]:mcmember['interests'][groups[gname]] for gname in shadowgroups} client.lists.members.update(list_id=list_id, subscriber_hash=mcmemberid, data={'interests' : merge_dicts(mcapi.nonmember, memberinterests)}) stat.nonmember += 1 # log stats thislogger.info ( stat )
def create_app(config_obj, configfiles=None, init_for_operation=True): ''' apply configuration object, then configuration files ''' global app app = Flask('members') app.config.from_object(config_obj) if configfiles: # backwards compatibility if type(configfiles) == str: configfiles = [configfiles] for configfile in configfiles: appconfig = getitems(configfile, 'app') app.config.update(appconfig) # tell jinja to remove linebreaks app.jinja_env.trim_blocks = True app.jinja_env.lstrip_blocks = True # define product name (don't import nav until after app.jinja_env.globals['_productname'] set) app.jinja_env.globals['_productname'] = app.config['THISAPP_PRODUCTNAME'] app.jinja_env.globals['_productname_text'] = app.config['THISAPP_PRODUCTNAME_TEXT'] for configkey in ['SECURITY_EMAIL_SUBJECT_PASSWORD_RESET', 'SECURITY_EMAIL_SUBJECT_PASSWORD_CHANGE_NOTICE', 'SECURITY_EMAIL_SUBJECT_PASSWORD_NOTICE', ]: app.config[configkey] = app.config[configkey].format(productname=app.config['THISAPP_PRODUCTNAME_TEXT']) # initialize database from .model import db db.init_app(app) # initialize uploads if init_for_operation: init_uploads(app) # handle <interest> in URL - https://flask.palletsprojects.com/en/1.1.x/patterns/urlprocessors/ @app.url_value_preprocessor def pull_interest(endpoint, values): try: g.interest = values.pop('interest', None) except AttributeError: g.interest = None finally: if not g.interest: g.interest = request.args.get('interest', None) # add loutilities tables-assets for js/css/template loading # see https://adambard.com/blog/fresh-flask-setup/ # and https://webassets.readthedocs.io/en/latest/environment.html#webassets.env.Environment.load_path # loutilities.__file__ is __init__.py file inside loutilities; os.path.split gets package directory loutilitiespath = os.path.join(os.path.split(loutilities.__file__)[0], 'tables-assets', 'static') @app.route('/loutilities/static/<path:filename>') def loutilities_static(filename): return send_from_directory(loutilitiespath, filename) # bring in js, css assets here, because app needs to be created first from .assets import asset_env, asset_bundles with app.app_context(): # needs to be set before update_local_tables called and before UserSecurity() instantiated g.loutility = Application.query.filter_by(application=app.config['APP_LOUTILITY']).one() # update LocalUser and LocalInterest tables if init_for_operation: update_local_tables() # js/css files asset_env.append_path(app.static_folder) asset_env.append_path(loutilitiespath, '/loutilities/static') # templates loader = ChoiceLoader([ app.jinja_loader, PackageLoader('loutilities', 'tables-assets/templates') ]) app.jinja_loader = loader # initialize assets asset_env.init_app(app) asset_env.register(asset_bundles) # Set up Flask-Mail [configuration in <application>.cfg] and security mailer mail = Mail(app) def security_send_mail(subject, recipient, template, **context): # this may be called from view which doesn't reference interest # if so pick up user's first interest to get from_email address if not g.interest: g.interest = context['user'].interests[0].interest if context['user'].interests else None if g.interest: from_email = localinterest().from_email # use default if user didn't have any interests else: from_email = current_app.config['SECURITY_EMAIL_SENDER'] # copied from flask_security.utils.send_mail if isinstance(from_email, LocalProxy): from_email = from_email._get_current_object() ctx = ('security/email', template) html = render_template('%s/%s.html' % ctx, **context) text = render_template('%s/%s.txt' % ctx, **context) sendmail(subject, from_email, recipient, html=html, text=text) # Set up Flask-Security global user_datastore, security user_datastore = SQLAlchemyUserDatastore(db, User, Role) security = UserSecurity(app, user_datastore, send_mail=security_send_mail) # activate views from .views import userrole as userroleviews from loutilities.user.views import bp as userrole app.register_blueprint(userrole) from .views.frontend import bp as frontend app.register_blueprint(frontend) from .views.admin import bp as admin app.register_blueprint(admin) # need to force app context else get # RuntimeError: Working outside of application context. # RuntimeError: Attempted to generate a URL without the application context being pushed. # see http://kronosapiens.github.io/blog/2014/08/14/understanding-contexts-in-flask.html with app.app_context(): # import navigation after views created from . import nav # turn on logging from .applogging import setlogging setlogging() # set up scoped session from sqlalchemy.orm import scoped_session, sessionmaker # see https://github.com/pallets/flask-sqlalchemy/blob/706982bb8a096220d29e5cef156950237753d89f/flask_sqlalchemy/__init__.py#L990 db.session = scoped_session(sessionmaker(autocommit=False, autoflush=False, binds=db.get_binds(app))) db.query = db.session.query_property() # handle favicon request for old browsers app.add_url_rule('/favicon.ico', endpoint='favicon', redirect_to=url_for('static', filename='favicon.ico')) # ---------------------------------------------------------------------- @app.before_request def before_request(): g.loutility = Application.query.filter_by(application=app.config['APP_LOUTILITY']).one() if current_user.is_authenticated: user = current_user email = user.email # used in layout.jinja2 app.jinja_env.globals['user_interests'] = sorted([{'interest': i.interest, 'description': i.description} for i in user.interests if g.loutility in i.applications], key=lambda a: a['description'].lower()) session['user_email'] = email else: # used in layout.jinja2 pubinterests = Interest.query.filter_by(public=True).all() app.jinja_env.globals['user_interests'] = sorted([{'interest': i.interest, 'description': i.description} for i in pubinterests if g.loutility in i.applications], key=lambda a: a['description'].lower()) session.pop('user_email', None) # ---------------------------------------------------------------------- @app.after_request def after_request(response): # # check if there are any changes needed to LocalUser table # userupdated = User.query.order_by(desc('updated_at')).first().updated_at # localuserupdated = LocalUser.query.order_by(desc('updated_at')).first().updated_at # interestupdated = Interest.query.order_by(desc('updated_at')).first().updated_at # localinterestupdated = LocalInterest.query.order_by(desc('updated_at')).first().updated_at # if userupdated > localuserupdated or interestupdated > localinterestupdated: # update_local_tables() if not app.config['DEBUG']: app.logger.info( '{}: {} {} {}'.format(request.remote_addr, request.method, request.url, response.status_code)) return response # app back to caller return app
from __future__ import absolute_import # standard import os.path # pypi from celery import Celery # homegrown from loutilities.configparser import getitems # note doe not config backend here else state does not come back # see https://stackoverflow.com/questions/25495613/celery-getting-started-not-able-to-retrieve-results-always-pending app = Celery('proj', include=['proj.tasks']) # pull in configuration, configuration file in parent dir from this file configpath = os.path.join(os.path.sep.join(os.path.dirname(__file__).split(os.path.sep)[:-1]), 'celerytest.cfg') config = getitems(configpath, 'celery') # Optional configuration, see the application user guide. app.config_from_object(config) if __name__ == '__main__': app.start()
# standard import os.path # pypi from flask import Flask from celery import Celery # homegrown from loutilities.configparser import getitems app = Flask(__name__) # pull in app and celery configuration, configuration file in parent dir from this file configpath = os.path.join( os.path.sep.join(os.path.dirname(__file__).split(os.path.sep)[:-1]), 'celerytest.cfg') appconfig = getitems(configpath, 'app') app.config.update(appconfig) celeryconfig = getitems(configpath, 'celery') celery = Celery('proj') celery.conf.update(celeryconfig) # import views after configuration import longtask import debug
def updatestravaclubactivitycache(): #---------------------------------------------------------------------- ''' script to update the strava club activity cache usage: updatestravaclubactivitycache [-h] [-v] cachefile clubname script to update the strava club activity cache positional arguments: cachefile pathname of file in which cache is saved clubname full name of club as known to strava optional arguments: -h, --help show this help message and exit -v, --version show program's version number and exit ''' descr = ''' script to update the strava club activity cache ''' parser = argparse.ArgumentParser(description=descr,formatter_class=argparse.RawDescriptionHelpFormatter, version='{0} {1}'.format('running',version.__version__)) parser.add_argument('cachefile', help="pathname of file in which cache is saved") parser.add_argument('clubname', help="full name of club as known to strava") parser.add_argument('--configfile', help='optional configuration filename', default=None) args = parser.parse_args() # let user know what is going on print 'Updating Strava club activity cache for "{}"'.format(args.clubname) # configuration file supplied -- pull credentials from the app section if args.configfile: from loutilities.configparser import getitems appconfig = getitems(args.configfile, 'app') stravakey = appconfig['STRAVAKEY'] # no configuration file, the credentials should be retrieved with loutilities.apikey else: stravakey = None # instantiate the Strava object, which opens the cache ss = Strava(args.cachefile, key=stravakey) # get the club id clubs = ss.getathleteclubs() clubid = None for club in clubs: if club['name'] == args.clubname: clubid = club['id'] break # error if we didn't find the club if not clubid: sys.exit('ERROR: club "{}" not found'.format(args.clubname)) # retrieve all the latest activities activities = ss.getclubactivities(clubid) numadded = ss.clubactivitycacheadded cachesize = ss.clubactivitycachesize # close the object, which saves the cache ss.close() # let user know how we did print ' update complete:' print ' {} activities received from Strava'.format(len(activities)) print ' added {} of these to cache'.format(numadded) print ' new cache size = {}'.format(cachesize)
def importmembers(configfile, debug=False, stats=False): #---------------------------------------------------------------------- ''' import member data to mailchimp configfile must have the following [mailchimp] RSU_CLUB: <runsignup club_id> RSU_KEY: <key from runsignup partnership> RSU_SECRET: <secret from runsignup partnership> MC_KEY: <api key from MailChimp> MC_LIST: <name of list of interest> MC_GROUPNAMES: groupname1,groupname2,... MC_SHADOWCATEGORY: <name of shadowcategory> * shadowcategory groups are used to show desired inclusion but the group itself under other categories can be toggled by subscriber * this is used to prevent the recipient from being added back to the group against their wishes * if a groupname is not also under shadowcategory, it is only ticked if the subscriber was not present in the list prior to import * this category's group names include all of the group names which are reserved for members MC_CURRMEMBERGROUP: <name of group which is set for current members> MC_PASTMEMBERGROUP: <name of group which is set for current and past members> :param configfile: name of configuration file :param debug: set to True for debug output :param stats: set to True for stats output (INFO) ''' # set up logging thislogger.propagate = True if debug: # set up debug logging thislogger.setLevel(logging.DEBUG) elif stats: # INFO logging thislogger.setLevel(logging.INFO) else: # WARNING logging thislogger.setLevel(logging.WARNING) # load configuration rsuconfig = getitems(configfile, 'runsignup') mcconfig = getitems(configfile, 'mailchimp') club_id = rsuconfig['RSU_CLUB'] rsukey = rsuconfig['RSU_KEY'] rsusecret = rsuconfig['RSU_SECRET'] mckey = mcconfig['MC_KEY'] mclist = mcconfig['MC_LIST'] mcgroupnames = mcconfig['MC_GROUPNAMES'].split(',') mcshadowcategory = mcconfig['MC_SHADOWCATEGORY'] mcpastmembergroupname = mcconfig['MC_PASTMEMBERGROUP'] mccurrmembergroupname = mcconfig['MC_CURRMEMBERGROUP'] mctimeout = float(mcconfig['MC_TIMEOUT']) # use Transform to simplify RunSignUp format xform = Transform( { 'last': lambda mem: mem['user']['last_name'], 'first': lambda mem: mem['user']['first_name'], 'email': lambda mem: mem['user']['email'] if 'email' in mem['user'] else '', 'primary': lambda mem: mem['primary_member'] == 'T', 'start': 'membership_start', 'end': 'membership_end', 'modified': 'last_modified', }, # source and target are dicts, not objects sourceattr=False, targetattr=False) # download current member list from RunSignUp # get current members from RunSignUp, transforming each to local format # only save one member per email address, primary member preferred rsu = RunSignUp(key=rsukey, secret=rsusecret, debug=debug) rsu.open() rsumembers = rsu.members(club_id) rsucurrmembers = {} for rsumember in rsumembers: memberrec = {} xform.transform(rsumember, memberrec) memberkey = memberrec['email'].lower() # only save if there's an email address # the primary member takes precedence, but if different email for nonprimary members save those as well if memberkey and (memberrec['primary'] or memberkey not in rsucurrmembers): rsucurrmembers[memberkey] = memberrec rsu.close() # It's important not to add someone back to a group which they've decided not to receive # emails from. For this reason, a membergroup is defined with the same group names as # the real groups the user is interested in, for those groups which don't make up the whole # list. # download categories / groups from MailChimp client = MailChimp(mc_api=mckey, timeout=mctimeout) lists = client.lists.all(get_all=True, fields="lists.name,lists.id") list_id = [lst['id'] for lst in lists['lists'] if lst['name'] == mclist][0] categories = client.lists.interest_categories.all( list_id=list_id, fields="categories.title,categories.id") # groups are for anyone, shadowgroups are for members only groups = {} shadowgroups = {} # for debugging allgroups = {} for category in categories['categories']: mcgroups = client.lists.interest_categories.interests.all( list_id=list_id, category_id=category['id'], fields="interests.name,interests.id") for group in mcgroups['interests']: # save for debug allgroups[group['id']] = '{} / {}'.format(category['title'], group['name']) # special group to track past members if group['name'] == mcpastmembergroupname: mcpastmembergroup = group['id'] # and current members elif group['name'] == mccurrmembergroupname: mccurrmembergroup = group['id'] # newly found members are enrolled in all groups elif category['title'] != mcshadowcategory: groups[group['name']] = group['id'] # shadowgroups is used to remember the state of member's only groups for previous members # if a member's membership has expired they must be removed from any group(s) which have the same name as # those within the shadowgroup(s) (associated groups) # additionally if any nonmembers are found, they must be removed from the associated groups # this last bit can happen if someone who is not a member tries to enroll in a members only group else: shadowgroups[group['name']] = group['id'] # set up specific groups for mc api mcapi = Obj() # members new to the list get all the groups mcapi.newmember = { id: True for id in list(groups.values()) + list(shadowgroups.values()) + [mccurrmembergroup] + [mcpastmembergroup] } # previous members who lapsed get the member groups disabled mcapi.nonmember = { id: False for id in [ groups[gname] for gname in list(groups.keys()) if gname in shadowgroups ] + [mccurrmembergroup] } # members groups set to True, for mcapi.unsubscribed merge mcapi.member = { id: True for id in list(shadowgroups.values()) + [ groups[gname] for gname in list(groups.keys()) if gname in shadowgroups ] + [mccurrmembergroup] + [mcpastmembergroup] } # unsubscribed members who previously were not past members get member groups turned on and 'other' groups turned off mcapi.unsubscribed = merge_dicts( mcapi.member, { id: False for id in [ groups[gname] for gname in list(groups.keys()) if gname not in shadowgroups ] }) # retrieve all members of this mailchimp list # key these into dict by id (md5 has of lower case email address) tmpmcmembers = client.lists.members.all( list_id=list_id, get_all=True, fields= 'members.id,members.email_address,members.status,members.merge_fields,members.interests' ) mcmembers = {} for mcmember in tmpmcmembers['members']: mcmembers[mcmember['id']] = mcmember # collect some stats stat = Stat([ 'addedtolist', 'newmemberunsubscribed', 'newmember', 'pastmember', 'nonmember', 'memberunsubscribedskipped', 'membercleanedskipped', 'mailchimperror' ]) # loop through club members # if club member is in mailchimp # make sure shadowgroups are set (but don't change groups as these may have been adjusted by club member) # don't change subscribed status # pop off mcmembers as we want to deal with the leftovers later # if club member is not already in mailchimp # add assuming all groups (groups + shadowgroups) for memberkey in rsucurrmembers: clubmember = rsucurrmembers[memberkey] mcmemberid = mcid(clubmember['email']) thislogger.debug('processing {} {}'.format(clubmember['email'], mcmemberid)) # if club member is in mailchimp if mcmemberid in mcmembers: mcmember = mcmembers.pop(mcmemberid) # check if any changes are required # change required if current member not set if not mcmember['interests'][mccurrmembergroup]: # if not past member, just set the needful if not mcmember['interests'][mcpastmembergroup]: # if subscribed, all groups are set if mcmember['status'] == 'subscribed': client.lists.members.update( list_id=list_id, subscriber_hash=mcmemberid, data={'interests': mcapi.newmember}) stat.newmember += 1 # if unsubscribed, subscribe them to member stuff, but remove everything else elif mcmember['status'] == 'unsubscribed': try: client.lists.members.update( list_id=list_id, subscriber_hash=mcmemberid, data={ 'interests': mcapi.unsubscribed, 'status': 'subscribed' }) stat.newmemberunsubscribed += 1 # MailChimp won't let us resubscribe this member except MailChimpError as e: thislogger.info( 'member unsubscribed, skipped: {}'.format( clubmember['email'])) stat.memberunsubscribedskipped += 1 # other statuses are skipped else: thislogger.info('member cleaned, skipped: {}'.format( clubmember['email'])) stat.membercleanedskipped += 1 # past member, recall what they had set before for the member stuff else: pastmemberinterests = merge_dicts( { groups[gname]: mcmember['interests'][shadowgroups[gname]] for gname in list(shadowgroups.keys()) }, {mccurrmembergroup: True}) client.lists.members.update( list_id=list_id, subscriber_hash=mcmemberid, data={'interests': pastmemberinterests}) stat.pastmember += 1 # if club member is missing from mailchimp else: try: client.lists.members.create(list_id=list_id, data={ 'email_address': clubmember['email'], 'merge_fields': { 'FNAME': clubmember['first'], 'LNAME': clubmember['last'] }, 'interests': mcapi.newmember, 'status': 'subscribed' }) stat.addedtolist += 1 except MailChimpError as e: ed = e.args[0] thislogger.warning('MailChimpError {} for {}: {}'.format( ed['title'], clubmember['email'], ed['detail'])) stat.mailchimperror += 1 # at this point, mcmembers have only those enrollees who are not in the club # loop through each of these and make sure club only interests are removed for mcmemberid in mcmembers: mcmember = mcmembers[mcmemberid] # change required if current member set if mcmember['interests'][mccurrmembergroup]: # save member interests for later if they rejoin memberinterests = { shadowgroups[gname]: mcmember['interests'][groups[gname]] for gname in shadowgroups } client.lists.members.update(list_id=list_id, subscriber_hash=mcmemberid, data={ 'interests': merge_dicts( mcapi.nonmember, memberinterests) }) stat.nonmember += 1 # log stats thislogger.info(stat)