def filters(org_label, project_label, filter_label): ''' creating filters for various purposes /organizations/aquaya/projects/water-quality/filters?create=true : create a new filter config, immediately redirect to editing /organizations/aquaya/projects/water-quality/filters/big-cities : view a filter /organizations/aquaya/projects/water-quality/filters/big-cities?edit=true : edit a filter; accepts GET or POST ''' user = User.objects(email=session['email'])[0] orgs = Organization.objects(label=org_label) if not orgs: flash('Organization "%s" not found, sorry!' % org_label, 'warning') return redirect(url_for('organizations')) org = orgs[0] # permission-check if org not in user.organizations and not user.admin_rights: app.logger.error('%s tried to view a project but was \ denied for want of admin rights' % session['email']) abort(404) # find the project projects = Project.objects(label=project_label, organization=org) if not projects: flash('Project "%s" not found, sorry!' % project_label, 'warning') return redirect(url_for('organizations', org_label=org.label)) project = projects[0] if request.method == 'POST': # we have a filter_label filters= Filter.objects(label=filter_label, project=project) if not filters: abort(404) # should try to avoid overriding the builtin.. filter = filters[0] form_type = request.form.get('form_type', '') if form_type == 'info': if filter.name != request.form.get('name', ''): name = request.form.get('name', '') filter.name = name filters = Filter.objects(project=project).only('label') labels = [f.label for f in filters] filter.label = utilities.generate_label(name, labels) filter.description = request.form.get('description', '') filter.comparison = request.form.get('comparison', '') # header of the format header_id__4abcd0001 header_id = request.form.get('header', '') header_id = header_id.split('header_id__')[1] header = Header.objects(id=header_id)[0] filter.header = header if header.data_type == 'datetime': # check if relative or absolute datetime comparison if filter.comparison in \ constants.filter_comparisons['datetime_absolute']: compare_to = '%s %s' % ( request.form.get('date_compare_to', '') , request.form.get('time_compare_to', '')) try: dt = datetime.datetime.strptime( compare_to, '%B %d, %Y %I:%M %p') filter.compare_to = {'value': dt} except: flash("didn't understand that date formatting; make \ sure it looks like 'June 21, 2012' and \ '02:45 PM'", 'error') return redirect(url_for('filters', org_label=org.label , project_label=project.label , filter_label=filter.label , edit='true')) elif filter.comparison in \ constants.filter_comparisons['datetime_relative']: inputs = [request.form.get('relative_years', '') , request.form.get('relative_months', '') , request.form.get('relative_weeks', '') , request.form.get('relative_days', '') , request.form.get('relative_hours', '') , request.form.get('relative_minutes', '')] parsed_inputs = [] for i in inputs: if i: parsed_inputs.append(i) else: parsed_inputs.append(0) # verify that we have numbers try: values = [int(i) for i in parsed_inputs] except: flash('input times must be whole numbers', 'error') return redirect(url_for('filters', org_label=org.label , project_label=project.label , filter_label=filter.label , edit='true')) filter.compare_to = {'value': values} else: app.logger.info('unknown comparison "%s" for project %s' \ % (comparison, project.name)) abort(404) else: # not datetime; cast the value we're comparing against compare_to = request.form.get('string_compare_to', '') filter.compare_to = {'value': utilities.smart_cast(compare_to)} elif form_type == 'admin': # delete the filter name = filter.name # pull filters out of the statistics statistics = Statistic.objects(filters=filter) for statistic in statistics: statistic.update(pull__filters=filter) # pull filters out of the graphs graphs = Graph.objects(filters=filter) for graph in graphs: graph.update(pull__filters=filter) filter.delete() app.logger.info('%s deleted filter "%s"' % \ (session['email'], name)) flash('filter "%s" was deleted successfully' % name, 'success') return redirect(url_for('filters', org_label=org.label , project_label=project.label)) else: # bad 'form_type' abort(404) try: filter.save() flash('changes saved successfully', 'success') return redirect(url_for('filters', org_label=org.label , project_label=project.label, filter_label=filter.label)) except: app.logger.error('%s experienced an error saving info about %s' % ( session['email'], request.form['name'])) flash('Error saving changes -- make sure filter names are unique.' , 'error') return redirect(url_for('filters', org_label=org.label , project_label=project.label, filter_label=filter_label , edit='true')) if request.method == 'GET': if filter_label: filters = Filter.objects(label=filter_label, project=project) if not filters: app.logger.error('%s tried to access a filter that does not \ exist' % session['email']) flash('Filter "%s" not found, sorry!' % filter_label , 'warning') return redirect(url_for('projects', org_label=org.label , project_label=project.label)) filter = filters[0] if request.args.get('edit', '') == 'true': # valid comparisons comparisons = json.dumps(constants.filter_comparisons) # list of allowed absolute datetime comparisons absolute_comparisons = \ constants.filter_comparisons['datetime_absolute'] relative_values = None if filter.comparison in \ constants.filter_comparisons['datetime_relative']: relative_values = filter.compare_to['value'] return render_template('filter_edit.html', filter=filter , comparisons=comparisons , absolute_datetime_comparisons=absolute_comparisons , relative_datetime_compare_values=relative_values) else: # apply the filter; start with some defaults conditions = { 'project': project , 'unique': True , 'visible': True } entries = Entry.objects(**conditions) project_count = len(entries) if not filter.comparison: return render_template('filter.html', filter=filter , project_count=project_count) # find all matches for this filter to display matches = utilities.apply_filter(filter, entries) # sort the matches matches = utilities.sort_entries(project, matches) # pagination entries_per_page = 10 pages = utilities.calculate_pages(len(matches) , entries_per_page=entries_per_page) # validate the requested page current_page = utilities.validate_page_request( request.args.get('page', 1), pages) # manually paginate start_index = entries_per_page * (current_page - 1) end_index = start_index + entries_per_page paginated_matches = matches[start_index:end_index] # list of allowed absolute datetime comparisons absolute_comparisons = \ constants.filter_comparisons['datetime_absolute'] # if the filter is for a relative datetime, parse the value relative_value = None if filter.comparison in \ constants.filter_comparisons['datetime_relative']: # filter.compare_to is a list [year, month, etc..] # first non zero value will be used for i in range(len(filter.compare_to['value'])): if filter.compare_to['value'][i] != 0: break # values in the compare_to list periods = ['year', 'month', 'week', 'day', 'hour' , 'minute'] # see if we need to make it plural suffix = '' if filter.compare_to['value'][i] > 1: suffix = 's' relative_value = '%s %s%s' \ % (filter.compare_to['value'][i], periods[i], suffix) return render_template('filter.html', filter=filter , entries = paginated_matches , total_matches = len(matches) , project_count=project_count , current_page = current_page , number_of_pages = pages , absolute_datetime_comparisons=absolute_comparisons , relative_datetime_value=relative_value) if request.args.get('create', '') == 'true': # create a new filter # CSRF validation token = request.args.get('token', '') if not verify_token(token): abort(403) try: filter_name = 'filter-%s' % utilities.generate_random_string(6) new_filter= Filter( creation_time = datetime.datetime.utcnow() , creator = user , label = filter_name.lower() , project = project , name = filter_name ) new_filter.save() app.logger.info('filter created by %s' % session['email']) flash('filter created; please change the defaults', 'success') except: app.logger.error('filter creation failed for %s' % \ session['email']) flash('There was an error, sorry :/', 'error') return redirect(url_for('projects', org_label=org.label , project=project.label)) # redirect to the editing screen return redirect(url_for('filters', org_label=org.label , project_label=project.label, filter_label=new_filter.label , edit='true')) # no filter in particular was specified, show 'em all filters = Filter.objects(project=project) # list of allowed absolute datetime comparisons absolute_comparisons = \ constants.filter_comparisons['datetime_absolute'] relative_values = {} for filter in filters: # if the filter is for a relative datetime, parse the value relative_value = None if filter.comparison in \ constants.filter_comparisons['datetime_relative']: # filter.compare_to is a list [year, month, etc..] # first non zero value will be used for i in range(len(filter.compare_to['value'])): if filter.compare_to['value'][i] != 0: break # values in the compare_to list periods = ['year', 'month', 'week', 'day', 'hour' , 'minute'] # see if we need to make it plural suffix = '' if filter.compare_to['value'][i] > 1: suffix = 's' relative_value = '%s %s%s' % ( filter.compare_to['value'][i], periods[i], suffix) relative_values[filter.name] = relative_value return render_template('project_filters.html', project=project , filters=filters , absolute_datetime_comparisons=absolute_comparisons , relative_datetime_values=relative_values)
def statistics(org_label, project_label, statistic_label): ''' creating statistics for various purposes /organizations/aquaya/projects/water-quality/statistics : viewing a list of this project's statistics /organizations/aquaya/projects/water-quality/statistics?create=true : create a new statistic config, immediately redirect to editing /organizations/aquaya/projects/water-quality/statistics/mean-pH : view a statistic /organizations/aquaya/projects/water-quality/statistics/mean-pH?edit=true : edit a statistic; accepts GET or POST ''' user = User.objects(email=session['email'])[0] orgs = Organization.objects(label=org_label) if not orgs: flash('Organization "%s" not found, sorry!' % org_label, 'warning') return redirect(url_for('organizations')) org = orgs[0] # permission-check if org not in user.organizations and not user.admin_rights: app.logger.error('%s tried to view a project but was \ denied for want of admin rights' % session['email']) abort(404) # find the project projects = Project.objects(label=project_label, organization=org) if not projects: flash('Project "%s" not found, sorry!' % project_label, 'warning') return redirect(url_for('organizations', org_label=org.label)) project = projects[0] if request.method == 'POST': # we have a statistic_label statistics = Statistic.objects(label=statistic_label, project=project) if not statistics: abort(404) statistic = statistics[0] form_type = request.form.get('form_type', '') if form_type == 'info': if statistic.name != request.form.get('name', ''): name = request.form.get('name', '') statistic.name = name statistics = Statistic.objects(project=project).only('label') labels = [s.label for s in statistics] statistic.label = utilities.generate_label(name, labels) statistic.description = request.form.get('description', '') statistic.statistic_type = request.form.get('statistic_type', '') # header is of the form 'header_id__4abcd0012' header = request.form.get('header', '') header_id = header.split('header_id__')[1] header = Header.objects(id=header_id)[0] statistic.header = header elif form_type == 'create-table': compute_multiple = request.form.get('compute_multiple', '') if compute_multiple == 'true': statistic.pivot = True else: statistic.pivot = False header_id = request.form.get('compute_multiple_header', '') headers = Header.objects(project=project, id=header_id) if not headers: abort(404) statistic.pivot_header = headers[0] # the name of the select with the values is based on header id values = request.form.getlist('header_unique_values') escaped_values = [bleach.clean(v) for v in values] statistic.pivot_values = escaped_values statistic.save() flash('Table settings saved successfully', 'success') return redirect(url_for('statistics' , org_label=org.label , project_label=project.label , statistic_label=statistic.label , edit='true')) elif form_type == 'filters': # extract the 'any filters' vs 'all' distinction filter_settings = request.form.get('apply_any_filters', '') if filter_settings == 'true': statistic.apply_any_filters = True else: statistic.apply_any_filters = False statistic.save() # attach filter to statistic requested_filter_ids = request.form.getlist('filters') attached_filters = [] for requested_id in requested_filter_ids: prefix, filter_id = requested_id.split('__') filters = Filter.objects(id=filter_id) if not filters: abort(404) attached_filters.append(filters[0]) statistic.update(set__filters = attached_filters) return redirect(url_for('statistics', org_label=org.label , project_label=project.label , statistic_label=statistic.label, edit='true')) elif form_type == 'admin': # delete the statistic name = statistic.name utilities.delete_statistic(statistic, session['email']) flash('statistic "%s" was deleted successfully' % name, 'success') return redirect(url_for('statistics', org_label=org.label , project_label=project.label)) else: # bad 'form_type' abort(404) try: statistic.save() flash('Changes saved successfully', 'success') return redirect(url_for('statistics', org_label=org.label , project_label=project.label , statistic_label=statistic.label, edit='true')) except: app.logger.error('%s experienced an error saving info about %s' % ( session['email'], request.form['name'])) flash('Error saving changes -- make sure statistic names are \ unique.', 'error') return redirect(url_for('statistics', org_label=org.label , project_label=project.label, statistic_label=statistic_label , edit='true')) if request.method == 'GET': if statistic_label: statistics = Statistic.objects(label=statistic_label , project=project) if not statistics: app.logger.error('%s tried to access a statistic that does \ not exist' % session['email']) flash('Statistic "%s" not found, sorry!' % statistic_label , 'warning') return redirect(url_for('projects', org_label=org.label , project_label=project.label)) statistic = statistics[0] if request.args.get('edit', '') == 'true': # valid statistic types statistic_types = constants.statistic_types available_filters = Filter.objects(project=project) # find all string-type headers # these are available for compute_multiple string_type_headers = [] for header in project.ordered_schema: if header.display and header.data_type == 'string': string_type_headers.append(header) return render_template('statistic_edit.html' , statistic = statistic , statistic_types = statistic_types , available_filters = available_filters , string_type_headers = string_type_headers) else: # apply the filters and show matches conditions = { 'project': project , 'unique': True , 'visible': True } entries = Entry.objects(**conditions) # sort the entries based on project defaults entries = utilities.sort_entries(project, entries) project_count = len(entries) for filter in statistic.filters: if not filter.comparison: flash('filter "%s" does not have a complete \ definition' % filter.name, 'warning') return render_template('statistic.html' , statistic=statistic , entries=entries , project_count=project_count) # apply relevant filters and then computes the statistic if statistic.pivot: # returns dict keyed by the statistic's pivot_values result = utilities.compute_pivot_statistic(statistic , entries) else: # returns a single result result = utilities.compute_statistic(statistic, entries) # find matches to the applied filters matches = utilities.apply_filters(statistic.filters, entries , statistic.apply_any_filters) total_matches = len(matches) # pagination entries_per_page = 10 pages = utilities.calculate_pages(len(matches) , entries_per_page=entries_per_page) # validate the requested page current_page = utilities.validate_page_request( request.args.get('page', 1), pages) # manually paginate start_index = entries_per_page * (current_page - 1) end_index = start_index + entries_per_page paginated_matches = matches[start_index:end_index] return render_template('statistic.html' , statistic=statistic , entries=paginated_matches , total_matches = total_matches , project_count=project_count , result=result , current_page = current_page , number_of_pages = pages) if request.args.get('create', '') == 'true': # create a new statistic # CSRF validation token = request.args.get('token', '') if not verify_token(token): abort(403) try: statistic_name = 'stat-%s' \ % utilities.generate_random_string(6) new_statistic= Statistic( creation_time = datetime.datetime.utcnow() , creator = user , label = statistic_name.lower() , project = project , name = statistic_name ) new_statistic.save() app.logger.info('statistic created by %s' % session['email']) flash('statistic created; please change the defaults' , 'success') except: app.logger.error('statistic creation failed for %s' % \ session['email']) flash('There was an error, sorry :/', 'error') return redirect(url_for('projects', org_label=org.label , project=project.label)) # redirect to the editing screen return redirect(url_for('statistics', org_label=org.label , project_label=project.label , statistic_label=new_statistic.label, edit='true')) # no statistic in particular was specified; show em all statistics = Statistic.objects(project=project) # calculate the results of these statistics statistic_results = {} unique_entries = Entry.objects(project=project, unique=True , visible=True) for statistic in statistics: statistic_results[statistic.name] = \ utilities.compute_statistic(statistic, unique_entries) return render_template('project_statistics.html', project=project , statistics=statistics, statistic_results=statistic_results)