def export_event_aggregates(): """ Exports the event aggregates as a csv """ database = Database() # Make sure the user has access to the module jwt_user = get_jwt_identity() user = database.get_item('users', jwt_user) authorized = conf.EVENT_GROUP in user['modules'] log_request(request, jwt_user, authorized) if not authorized: response = {'message': '%s does not have acccess to events'%(jwt_user)} return jsonify(response), 403 q = request.args.get('q') query = ('name', q) if q else None database = Database() df = database.read_table('event_aggregates', query=query) today = str(datetime.datetime.now())[:10] buffer = StringIO() df.to_csv(buffer, encoding='utf-8', index=False) output = make_response(buffer.getvalue()) output.headers["Content-Disposition"] = "attachment; filename=export.csv" output.headers["Content-type"] = "text/csv" return output
def test_load_items(): database = Database() database.delete_item('members', 'testid1') database.delete_item('members', 'testid2') columns = database.get_columns('members') item1 = {x: None for x in columns} item1['id'] = 'testid1' item2 = {x: None for x in columns} item2['id'] = 'testid2' items = [item1, item2] database.load_items(items, 'members') item1_ = database.get_item('members', 'testid1') assert item1_['id'] == 'testid1' item2_ = database.get_item('members', 'testid2') assert item2_['id'] == 'testid2' database.delete_item('members', 'testid1') item1_ = database.get_item('members', 'testid1') assert item1_ == None database.delete_item('members', 'testid2') item2_ = database.get_item('members', 'testid2') assert item2_ == None
def map_authorize(): """ Checks whether the users is authorized to view the map """ database = Database() jwt_user = get_jwt_identity() user = database.get_item('users', jwt_user) authorized = conf.MAP_GROUP in user['modules'] log_request(request, jwt_user, authorized) if not authorized: response = {'message': '%s does not have access the map' % (jwt_user)} return jsonify(response), 403 else: response = {'message': '%s is authorized to view the map' % (jwt_user)} return jsonify(response), 200
def member_authorize(): """ Checks to see if the user is authorized to see members """ database = Database() jwt_user = get_jwt_identity() user = database.get_item('users', jwt_user) authorized = conf.TRENDS_GROUP in user['modules'] log_request(request, jwt_user, authorized) if not authorized: response = { 'message': '%s does not have access to trends' % (jwt_user) } return jsonify(response), 403 else: del user['password'] del user['temporary_password'] return jsonify(user), 200
class EventbriteLoader(object): """Loads data from Eventbrite into Postgres """ def __init__(self, eventbrite_org, database=None): daiquiri.setup(level=logging.INFO) self.logger = daiquiri.getLogger(__name__) self.database = Database(database=database) self.eventbrite = Eventbrite() self.eventbrite_org = eventbrite_org def run(self, test=False): """ Runs the data load process """ last_load_date = self.database.last_event_load_date() if last_load_date: look_back = datetime.datetime.now() - datetime.timedelta(days=1) first_event = min(look_back, last_load_date) start = str(first_event)[:10] self.logger.info('Loading events starting at %s' % (start)) else: self.logger.info('Loading events from the first available event') start = None events = self.get_events(start=start, page=1) num_events = events['pagination']['object_count'] if num_events > 0: self.logger.info('There are %s events to process' % (num_events)) else: self.logger.info('There are not next events. Exiting') return more_events = True while more_events: for event in events['events']: if not event: continue msg = "Loading information for %s" % (event['name']['text']) self.logger.info(msg) # Load the event into the database. Delete the current # entry in order to maintain the unique index event_id = event['id'] if not test: self.database.delete_item('events', event_id) self.load_event(event) # Load the venue, if it does not already # appear in the database venue_id = event['venue_id'] venue_ = self.database.get_item('venues', venue_id) if venue_id and not venue_: venue = self.get_venue(venue_id) if not test: self.load_venue(venue) attendees = self.get_attendees(event_id, page=1) more_attendees = True while more_attendees: if not attendees: break for attendee in attendees['attendees']: if not attendee: continue if not test: self.database.delete_item('attendees', attendee['id'], {'event_id': event_id}) self.load_attendee(attendee) if test or not attendees['pagination']['has_more_items']: more_attendees = False break else: page = attendees['pagination']['page_number'] + 1 attendees = self.get_attendees(event_id, page) # Sleep to avoid the Eventbrite rate limit if test: return else: time.sleep(60) if not events['pagination']['has_more_items']: more_events = False break else: page = events['pagination']['page_number'] + 1 msg = 'Pulling events on page %s' % (page) self.logger.info(msg) events = self.get_events(start, page) def get_events(self, start, page=1): """ Pulls events from eventbrite and sleeps if the rate limit has been exceeded """ org_id = self.eventbrite_org events = self.eventbrite.get_events(org_id=org_id, start=start, page=page) if not events: # Sleep until eventbrite resets self.logger.info('Rate limit exceed. Sleeping 30 mins') time.sleep(3600) events = self.eventbrite.get_events(start=start, page=page) return events def get_attendees(self, event_id, page=1): """ Pulls attendees from eventbrite and sleeps if the rate limit has been exceeded """ attendees = self.eventbrite.get_attendees(event_id, page) if not attendees: # If events comes back as none, sleep until the # Eventbrite rate limit resets self.logger.info('Rate limit exceed. Sleeping 30 mins') time.sleep(3600) attendees = self.eventbrite.get_attendees(event_id, page) return attendees def get_venue(self, venue_id, page=1): """ Pull a venue and sleeps if the rate limit has been exceeded """ venue = self.eventbrite.get_venue(venue_id, page) if not venue: self.logger.info('Rate limit exceed. Sleeping 30 mins') time.sleep(3600) venue = self.eventbrite.get_venue(event_id, page) return venue def load_event(self, event): """ Loads an event into the database """ event_ = deepcopy(event) start = arrow.get(event_['start']['utc']).datetime event_['start_datetime'] = start end = arrow.get(event_['end']['utc']).datetime event_['end_datetime'] = end description = event_['description']['text'] event_['description'] = description name = event_['name']['text'] event_['name'] = name event_['load_datetime'] = datetime.datetime.utcnow() self.database.load_item(event_, 'events') def load_attendee(self, attendee): """ Loads an attendee into the database """ attendee_ = deepcopy(attendee) profile = attendee_['profile'] if 'name' in profile: attendee_['name'] = profile['name'] if 'first_name' in profile: attendee_['first_name'] = profile['first_name'] if 'last_name' in profile: attendee_['last_name'] = profile['last_name'] if 'email' in profile: attendee_['email'] = profile['email'] cost = attendee_['costs']['gross']['major_value'] attendee_['cost'] = float(cost) attendee_['load_datetime'] = datetime.datetime.utcnow() self.database.load_item(attendee_, 'attendees') def load_order(self, order): """ Loads an order into the database """ order_ = deepcopy(order) cost = order_['costs']['gross']['major_value'] order_['cost'] = float(cost) order_['load_datetime'] = datetime.datetime.utcnow() self.database.load_item(order_, 'orders') def load_venue(self, venue): """ Loads a venue into the database """ venue_ = deepcopy(venue) for key in venue_['address']: val = venue_['address'][key] venue_[key] = val venue_['latitude'] = float(venue_['latitude']) venue_['longitude'] = float(venue_['longitude']) self.database.load_item(venue_, 'venues')