Beispiel #1
0
def uploads(o_label, p_label, u_label):
    ''' uploadin
    /organizations/aquaya/projects/water-quality/uploads?create=true
        : create a new upload, immediately redirect to editing
    /organizations/aquaya/projects/water-quality/uploads/may-2012
        : view an upload
    /organizations/aquaya/projects/water-quality/uploads/may-2012?edit=true
        : edit an upload; accepts GET or POST
    '''
    # can't wrap the line through a route or spaces are injected
    # hence this silliness:
    org_label = o_label
    project_label = p_label
    upload_label = u_label

    user = User.objects(email=session['email'])[0]

    orgs = Organization.objects(label=org_label)
    if not orgs:
        flash('Organization "%s" not found, sorry!' % org_label, 'warning')
        return redirect(url_for('organizations'))
    org = orgs[0]
    
    # permission-check
    if org not in user.organizations and not user.admin_rights:
        app.logger.error('%s tried to view a project but was \
            denied for want of admin rights' % session['email'])
        abort(404)
    
    projects = Project.objects(label=project_label, organization=org) 
    if not projects:
        flash('Project "%s" not found, sorry!' % project_label, 'warning')
        return redirect(url_for('organizations', org_label=org.label))
    project = projects[0]

    if request.method == 'GET':
        if not upload_label and request.args.get('create', '') == 'true':
            # create a new upload

            # CSRF validation
            token = request.args.get('token', '')
            if not verify_token(token):
                abort(403)

            try:
                name = 'upl-%s' % utilities.generate_random_string(6)
                new_upload = Upload(
                    label = name.lower()
                    , name = name
                    , project = project
                )
                new_upload.save()

                project.update(push__uploads=new_upload)
                app.logger.info('upload created by %s' % session['email'])
                flash('please select a file and make other edits', 'warning')
            except:
                app.logger.error('upload creation failed for %s' % \
                    session['email'])
                flash('There was an error, sorry :/', 'error')
                return redirect(url_for('projects', org_label=org.label
                    , project_label=project.label))
            
            # redirect to the editing screen
            return redirect(url_for('uploads', o_label=org.label
                , p_label=project.label, u_label=new_upload.label
                , edit='true'))

        elif not upload_label:
            # could show projects uploads but punting for now
            unique_entries = Entry.objects(project=project, unique=True
                , visible=True)
            duplicate_entries = Entry.objects(project=project
                , unique=False)
            hidden_entries = Entry.objects(project=project, visible=False
                , unique=True)
            
            upload_entry_counts = {}
            for upload in project.uploads:
                upload_entry_counts[upload.id] = Entry.objects(upload=upload).count()
            
            return render_template('project_uploads.html', project=project
                , unique_entries=unique_entries
                , duplicate_entries=duplicate_entries
                , upload_entry_counts=upload_entry_counts
                , hidden_entries=hidden_entries)
        
        else:
            # we have an upload label
            uploads = Upload.objects(label=upload_label, project=project) 
            if not uploads:
                flash('Upload "%s" not found, sorry!' % upload_label
                    , 'warning')
                return redirect(url_for('projects', org_label=org.label
                    , project_label=project.label))
            upload = uploads[0]

            if request.args.get('edit', '') == 'true':
                # show the editor

                # queue up any flashed messages
                # these get set during the worker's processing of a file
                worker_messages = upload.worker_messages
                for message in worker_messages:
                    flash(message['message'], message['status'])
                upload.update(set__worker_messages = [])

                return render_template('upload_edit.html', upload=upload
                        , number_of_pages=0)

            else:
                if not upload.filename:
                    # need to add one so redirect to editing
                    flash('please specify a file for this upload', 'warning')
                    return redirect(url_for('uploads', o_label=org.label
                        , p_label=project.label, u_label=upload.label
                        , edit='true'))
                else:

                    # things are still in progress
                    if not upload.s3_key:
                        flash('Your file is being processed in the '
                            'background.  Refresh the page to see updates.'
                            , 'info')
                        return render_template('upload.html', upload=upload
                                , number_of_pages=0)

                    # otherwise upload is complete, show the data
                    # pagination
                    total_entries = Entry.objects(project=project
                        , upload=upload).count()
                    entries_per_page = 10
                    pages = utilities.calculate_pages(total_entries
                        , entries_per_page=entries_per_page)

                    # validate the requested page
                    current_page = utilities.validate_page_request(
                        request.args.get('page', 1), pages)

                    # get the sorted entries
                    entries = utilities.query_entries(project
                        , upload = upload
                        , skip = (entries_per_page * (current_page - 1))
                        , limit = entries_per_page)
                    
                    hidden_entries_count = Entry.objects(upload=upload
                        , visible=False).count()

                    # queue up any flashed messages
                    # these get set during the worker's processing of a file
                    worker_messages = upload.worker_messages
                    for message in worker_messages:
                        flash(message['message'], message['status'])
                    upload.update(set__worker_messages = [])

                    return render_template('upload.html'
                        , upload=upload
                        , entries=entries
                        , total_entries = total_entries
                        , hidden_entries_count=hidden_entries_count
                        , current_page = current_page
                        , number_of_pages = pages)

    elif request.method == 'POST':
        # we have an upload label
        uploads = Upload.objects(label=upload_label, project=project)
        if not uploads:
            abort(404)
        upload = uploads[0]

        form_type = request.form.get('form_type', '')
        if form_type == 'info':

            if upload.name != request.form.get('name', ''):
                name = request.form.get('name', '')
                upload.update(set__name = name)

                uploads = Upload.objects(project=project).only('label')
                labels = [u.label for u in uploads]
                upload.update(set__label = 
                        utilities.generate_label(name, labels))

                upload.project.update(set__update_time =
                        datetime.datetime.utcnow())
                upload.reload()
            
            if upload.description != request.form.get('description', ''):
                upload.update(set__description =
                        request.form.get('description', ''))
                upload.project.update(set__update_time =
                        datetime.datetime.utcnow())

            if request.files.get('data_file', ''):
                data_file = request.files.get('data_file')
                
                try:
                    filename = uploaded_data.save(data_file)
                    if '..' in filename or filename.startswith('/'):
                        app.logger.info('%s tried to upload a malicious file \
                            "%s"' % (session['email'], filename))
                        flash('bad filename, sorry :/', 'error')
                        return redirect(url_for('uploads'
                            , o_label=org.label, p_label=project.label
                            , u_label=upload.label, edit='true'))

                    absolute_filename = uploaded_data.path(filename)
                
                except UploadNotAllowed:
                    app.logger.info('%s tried to upload an unsupported file'
                        % session['email'])
                    flash('we currently only support .xls files, sorry :/'
                        , 'error')
                    return redirect(url_for('uploads'
                        , o_label=org.label, p_label=project.label
                        , u_label=upload.label, edit='true'))
                
                upload.update(set__filename = 
                        os.path.basename(absolute_filename))
                upload.update(set__uploaded_by = user)

                # enqueue upload-processing
                redis_config = app.config['REDIS_CONFIG']
                use_connection(Redis(redis_config['host'], redis_config['port']
                        , password=redis_config['password']))
                queue = Queue()
                queue.enqueue(process_uploaded_file, upload.id,
                        absolute_filename)
                    
        elif form_type == 'admin':
            # delete the upload
            name = upload.name
            # pull out the upload from the parent project first
            project.update(pull__uploads=upload)
            
            # delete associated entries, remove from system, remove from s3
            deleted_entries_count = utilities.delete_upload(upload
                , session['email'])
            flash('upload "%s" was deleted successfully; %s entries deleted \
                as well' % (name, deleted_entries_count), 'success')
            
            # see if, given this deletion, the dupes are still dupes
            unduped_count = utilities.recheck_duplicate_entries(project)
            if unduped_count:
                flash('%s entries were automatically added to the project \
                    as they are no longer duplicates' % unduped_count, 'info')

            return redirect(url_for('uploads', o_label=org.label
                , p_label=project.label))
        
        else:
            # bad 'form_type'
            abort(404)
       
        try:
            upload.save()
            project.save()
        except:
            app.logger.error('%s experienced an error saving info about the \
                upload %s' % (session['email'], request.form['name']))
            flash('error, make sure names are unique', 'error')
            return redirect(url_for('projects'
                , org_label=upload.project.organization.label
                , project_label=upload.project.label))
        
        return redirect(url_for('uploads'
            , o_label=upload.project.organization.label, p_label=project.label
            , u_label=upload.label))
Beispiel #2
0
def entries(org_label, project_label, entry_id):
    ''' show a specific entry mostly for editing purposes
    '''
    user = User.objects(email=session['email'])[0]
    
    # find the org
    orgs = Organization.objects(label=org_label)
    if not orgs:
        flash('Organization "%s" not found, sorry!' % org_label, 'warning')
        return redirect(url_for('organizations'))
    org = orgs[0]
    
    # permission-check
    if org not in user.organizations and not user.admin_rights:
        app.logger.error('%s tried to view a project but was \
            denied for want of admin rights' % session['email'])
        abort(404)
    
    # find the project
    projects = Project.objects(label=project_label, organization=org) 
    if not projects:
        flash('Project "%s" not found, sorry!' % project_label, 'warning')
        return redirect(url_for('organizations', org_label=org.label))
    project = projects[0]

    if not entry_id:
        if request.method == 'POST':
            # downloading entries with applied filters
            filter_labels = request.form.getlist('filters')
            apply_any_filters = request.form.get('apply_any_filters', '')
            if apply_any_filters == 'true':
                apply_any_filters = True
            else:
                apply_any_filters = False

            # make a list of filter objects
            filters = []
            for filter_label in filter_labels:
                filters.append(Filter.objects(label=filter_label
                    , project=project)[0])
            
            # serve up a file of the project entries
            absolute_filename = utilities.download_all_entries(project
                , filters, format='xls', apply_any_filters=apply_any_filters)

            # delay the deletion so we have time to serve the file
            redis_config = app.config['REDIS_CONFIG']
            use_connection(Redis(redis_config['host'], redis_config['port']
                    , password=redis_config['password']))
            scheduler = Scheduler()
            scheduler.enqueue_in(datetime.timedelta(seconds=60)
                , delete_local_file, absolute_filename)

            return send_file(absolute_filename, as_attachment=True)
        

        '''GET request..
        display entries for the project
        '''
        duplicate_entries_count = Entry.objects(project=project
            , unique=False).count()
        hidden_entries_count = Entry.objects(project=project, unique=True
            , visible=False).count()
        unique_entries_count = Entry.objects(project=project, unique=True
            , visible=True).count()

        entry_type = request.args.get('type', '')

        if entry_type == 'duplicate':
            # show unique=False entries
            count = duplicate_entries_count
            unique = False
            visible = None
            template = 'project_duplicate_entries.html'

        elif entry_type == 'hidden':
            # show visible=False, unique=True entries
            count = hidden_entries_count
            unique = True
            visible = False
            template = 'project_hidden_entries.html'

        else:
            # show uniques
            count = unique_entries_count
            unique = True
            visible = True
            template = 'project_entries.html'

        entries_per_page = 10
        pages = utilities.calculate_pages(count
            , entries_per_page=entries_per_page)

        # validate the requested page
        current_page = utilities.validate_page_request(
            request.args.get('page', 1), pages)

        entries = utilities.query_entries(project
            , unique=unique
            , visible=visible
            , skip=(entries_per_page * (current_page - 1))
            , limit=entries_per_page)

        # present some filters if data is downloaded
        available_filters = Filter.objects(project=project)

        return render_template(template
            , project=project
            , entries=entries
            , unique_entries_count=unique_entries_count
            , duplicate_entries_count=duplicate_entries_count
            , hidden_entries_count=hidden_entries_count
            , available_filters=available_filters
            , current_page = current_page
            , number_of_pages = pages)
    

    # we have an entry_id, try to find the object
    entries = Entry.objects(id=entry_id)
    if not entries:
        flash('Entry "%s" not found, sorry!' % entry_id, 'warning')
        return redirect(url_for('entries', org_label=org.label
            , project_label=project.label))
    entry = entries[0]

    if request.method == 'GET':
        if request.args.get('edit', '') == 'true':
            return render_template('entry_edit.html', entry=entry)
        else:
            return render_template('entry.html', entry=entry)

    elif request.method == 'POST':
        form_type = request.form.get('form_type', '')

        if form_type == 'info':
            # track all modifications to this entry
            modifications = []
            # don't think I can set just one value in the dict with set__
            # so let's make a copy then alter it, then update it
            values = dict(entry.values)

            for header in entry.project.ordered_schema:
                if header.data_type == 'datetime':
                    full_dt = '%s %s' % (
                            request.form.get('%s__date' % header.id, '')
                            , request.form.get('%s__time' % header.id, ''))
                    try:
                        struct = time.strptime(full_dt, '%B %d, %Y %I:%M %p')
                        edited_val = datetime.datetime.fromtimestamp(
                                time.mktime(struct))
                    except ValueError:
                        # requested change was improperly formatted
                        message = ('Error.  Date-time data expected for the'
                                ' field "%s."' % header.label)
                        return redirect_to_editing(entry, 'error', message)

                elif header.data_type == 'number':
                    try:
                        edited_val = float(request.form.get(
                            str(header.id), ''))
                    except ValueError:
                        # requested change wasn't a number
                        message = ('Error.  Numerical data expected for the'
                                ' field "%s."' % header.label)
                        return redirect_to_editing(entry, 'error', message)

                else:
                    edited_val = request.form.get(str(header.id), '')

                # values that were originally None will show up here as ''
                if edited_val == '' and entry.values[header.name] == None:
                    continue

                if edited_val != entry.values[header.name]:
                    values[header.name] = edited_val
                    modifications.append('updated "%s" from "%s" to "%s"' %
                            (header.label, entry.values[header.name]
                                , edited_val))

            if modifications:
                ''' update the entry
                '''
                entry.update(set__values = values)
                entry.update(set__was_never_edited = False)

                ''' generate some hashes
                used to check other possible duplicative and hidden entries
                '''
                old_hash = str(entry.value_hash)
                # compute a hash for the new values
                m = hashlib.md5()
                #sorted_headers = [h.name for h in project.ordered_schema]
                sorted_headers = values.keys()
                sorted_headers.sort()
                for header in sorted_headers:
                    value = values[header]
                    if type(value) == type(u'unicode'):
                        m.update(value.encode('utf8'))
                    else:
                        m.update(str(value))
                new_hash = m.hexdigest()

                # if the old entry was unique and had a dupe..
                # ..that dupe is now unique
                # limit one to flip just one of several possible dupes
                if entry.unique:
                    old_dupes = Entry.objects(project=project, unique=False
                            , value_hash=old_hash).limit(1)
                    if old_dupes:
                        flash('The entry you have edited had a duplicate in'
                            ' the system.  That duplicate has now been marked'
                            ' "unique."', 'success')
                        old_dupes[0].update(set__unique = True)

                # process hidden entries
                # if there are hidden entries with these values..
                # ..sound the alarm
                hidden_entries = Entry.objects(project=project
                        , visible=False).only('value_hash')
                hidden_hashes = [h['value_hash'] for h in hidden_entries]
                if entry.visible and new_hash in hidden_hashes:
                    flash('Warning: an entry with these values was previously'
                            ' "hidden," i.e. removed from analysis.  Consider'
                            ' hiding this entry.', 'warning')

                # if the entry was hidden, remind the user of that fact..
                # since the values were just edited they may want to change it
                if not entry.visible:
                    flash('This entry is currently "hidden" and not included'
                        ' in analysis.  Consider un-hiding it to include your'
                        ' new edits.', 'warning')

                # search for unique and duplicate values
                uniques = Entry.objects(project=project
                        , unique=True).only('value_hash')
                unique_hashes = [u['value_hash'] for u in uniques]

                # if entry /was/ unique
                if entry.unique:
                    if new_hash in unique_hashes:
                        flash('This entry is now a duplicate of another entry'
                            ' in the system.', 'warning')
                        entry.update(set__unique=False)

                # entry /wasn't/ unique
                else:
                    if new_hash not in unique_hashes:
                        flash('This entry was formerly a duplicate but is now'
                            ' unique.', 'success')
                        entry.update(set__unique=True)

                entry.update(set__value_hash = new_hash)


                ''' create a comment encapsulating the changes
                '''
                new_comment = Comment(
                    body = '; '.join(modifications)
                    , creation_time = datetime.datetime.utcnow()
                    , editable = False
                    , entry = entry
                    , owner = user
                    , project = project
                )
                new_comment.save()

                message = ('Changes saved successfully: %s' %
                        '; '.join(modifications))
                return redirect_to_editing(entry, 'success', message)

            else:
                # no modifications made to the entry
                return redirect_to_editing(entry, None, None)


        elif form_type == 'hide_entry':
            # flip the 'visible' state of this entry
            if entry.visible:
                entry.update(set__visible = False)
                modifications = 'entry removed from analysis'
            else:
                entry.update(set__visible = True)
                modifications = 'entry re-included in analysis'
            
            # also find all duplicates in the project and hide/unhide them as well
            duplicate_entries = Entry.objects(project=project
                , value_hash=entry.value_hash)
            # if we have more than the original entry
            if len(duplicate_entries) > 1:
                for duplicate_entry in duplicate_entries:
                    if duplicate_entry.id == entry.id:
                        continue
                    entry.reload()
                    if entry.visible:
                        duplicate_entry.update(set__visible = True)
                    else:
                        duplicate_entry.update(set__visible = False)
                
                # append a note about the dupes
                # plural..
                dupes = len(duplicate_entries) - 1
                if dupes > 1:
                    plural = 's'
                else:
                    plural = ''
                modifications += ' with %s duplicate%s' % (dupes, plural)

            ''' create a comment encapsulating the changes
            hm, comments won't be attached to duplicates..alas
            '''
            new_comment = Comment(
                body = modifications
                , creation_time = datetime.datetime.utcnow()
                , editable = False
                , entry = entry
                , owner = user
                , project = project
            )
            new_comment.save()

            message = 'Changes saved successfully: %s.' % modifications
            return redirect_to_editing(entry, 'success', message)
Beispiel #3
0
def filters(org_label, project_label, filter_label):
    ''' creating filters for various purposes
    /organizations/aquaya/projects/water-quality/filters?create=true
        : create a new filter config, immediately redirect to editing
    /organizations/aquaya/projects/water-quality/filters/big-cities
        : view a filter
    /organizations/aquaya/projects/water-quality/filters/big-cities?edit=true
        : edit a filter; accepts GET or POST
    '''
    user = User.objects(email=session['email'])[0]
    
    orgs = Organization.objects(label=org_label)
    if not orgs:
        flash('Organization "%s" not found, sorry!' % org_label, 'warning')
        return redirect(url_for('organizations'))
    org = orgs[0]

    # permission-check
    if org not in user.organizations and not user.admin_rights:
        app.logger.error('%s tried to view a project but was \
            denied for want of admin rights' % session['email'])
        abort(404)
    
    # find the project
    projects = Project.objects(label=project_label, organization=org) 
    if not projects:
        flash('Project "%s" not found, sorry!' % project_label, 'warning')
        return redirect(url_for('organizations', org_label=org.label))
    project = projects[0]

    if request.method == 'POST':
        # we have a filter_label
        filters= Filter.objects(label=filter_label, project=project)
        if not filters:
            abort(404)
        # should try to avoid overriding the builtin..
        filter = filters[0]

        form_type = request.form.get('form_type', '')
        if form_type == 'info':
            if filter.name != request.form.get('name', ''):
                name = request.form.get('name', '')
                filter.name = name

                filters = Filter.objects(project=project).only('label')
                labels = [f.label for f in filters]
                filter.label = utilities.generate_label(name, labels)

            filter.description = request.form.get('description', '')
            filter.comparison = request.form.get('comparison', '')
            
            # header of the format header_id__4abcd0001
            header_id = request.form.get('header', '')
            header_id = header_id.split('header_id__')[1]
            header = Header.objects(id=header_id)[0]
            filter.header = header

            if header.data_type == 'datetime':
                # check if relative or absolute datetime comparison
                if filter.comparison in \
                    constants.filter_comparisons['datetime_absolute']:
                    compare_to = '%s %s' % (
                        request.form.get('date_compare_to', '')
                        , request.form.get('time_compare_to', ''))
                    try:
                        dt = datetime.datetime.strptime(
                            compare_to, '%B %d, %Y %I:%M %p')
                        filter.compare_to = {'value': dt}
                    except:
                        flash("didn't understand that date formatting; make \
                            sure it looks like 'June 21, 2012' and \
                            '02:45 PM'", 'error')
                        return redirect(url_for('filters', org_label=org.label
                            , project_label=project.label
                            , filter_label=filter.label
                            , edit='true'))
                elif filter.comparison in \
                    constants.filter_comparisons['datetime_relative']:

                    inputs = [request.form.get('relative_years', '')
                            , request.form.get('relative_months', '')
                            , request.form.get('relative_weeks', '')
                            , request.form.get('relative_days', '')
                            , request.form.get('relative_hours', '')
                            , request.form.get('relative_minutes', '')]

                    parsed_inputs = []
                    for i in inputs:
                        if i:
                            parsed_inputs.append(i)
                        else:
                            parsed_inputs.append(0)

                    # verify that we have numbers
                    try:
                        values = [int(i) for i in parsed_inputs]
                    except:
                        flash('input times must be whole numbers', 'error')
                        return redirect(url_for('filters', org_label=org.label
                            , project_label=project.label
                            , filter_label=filter.label
                            , edit='true'))

                    filter.compare_to = {'value': values}

                else:
                    app.logger.info('unknown comparison "%s" for project %s' \
                        % (comparison, project.name))
                    abort(404)

            else:
                # not datetime; cast the value we're comparing against
                compare_to = request.form.get('string_compare_to', '')
                filter.compare_to = {'value': utilities.smart_cast(compare_to)}

        elif form_type == 'admin':
            # delete the filter
            name = filter.name

            # pull filters out of the statistics
            statistics = Statistic.objects(filters=filter)
            for statistic in statistics:
                statistic.update(pull__filters=filter)

            # pull filters out of the graphs
            graphs = Graph.objects(filters=filter)
            for graph in graphs:
                graph.update(pull__filters=filter)

            filter.delete()
            app.logger.info('%s deleted filter "%s"' % \
                (session['email'], name))
            flash('filter "%s" was deleted successfully' % name, 'success')
            return redirect(url_for('filters', org_label=org.label
                , project_label=project.label))
        
        else:
            # bad 'form_type'
            abort(404)
       
        try:
            filter.save()
            flash('changes saved successfully', 'success')
            return redirect(url_for('filters', org_label=org.label
                , project_label=project.label, filter_label=filter.label))
        except:
            app.logger.error('%s experienced an error saving info about %s' % (
                session['email'], request.form['name']))
            flash('Error saving changes -- make sure filter names are unique.'
                , 'error')
            return redirect(url_for('filters', org_label=org.label
                , project_label=project.label, filter_label=filter_label
                , edit='true'))
        
    
    if request.method == 'GET':
        if filter_label:
            filters = Filter.objects(label=filter_label, project=project)
            if not filters:
                app.logger.error('%s tried to access a filter that does not \
                    exist' % session['email'])
                flash('Filter "%s" not found, sorry!' % filter_label
                    , 'warning')
                return redirect(url_for('projects', org_label=org.label
                    , project_label=project.label))
            filter = filters[0]

            if request.args.get('edit', '') == 'true':
                # valid comparisons
                comparisons = json.dumps(constants.filter_comparisons)

                # list of allowed absolute datetime comparisons
                absolute_comparisons = \
                    constants.filter_comparisons['datetime_absolute']

                relative_values = None
                if filter.comparison in \
                    constants.filter_comparisons['datetime_relative']:
                    relative_values = filter.compare_to['value']

                return render_template('filter_edit.html', filter=filter
                    , comparisons=comparisons
                    , absolute_datetime_comparisons=absolute_comparisons
                    , relative_datetime_compare_values=relative_values)
            else:
                # apply the filter; start with some defaults
                conditions = {
                    'project': project
                    , 'unique': True
                    , 'visible': True
                }
                entries = Entry.objects(**conditions)
                project_count = len(entries)
                
                if not filter.comparison:
                    return render_template('filter.html', filter=filter
                        , project_count=project_count)
                
                # find all matches for this filter to display
                matches = utilities.apply_filter(filter, entries)

                # sort the matches
                matches = utilities.sort_entries(project, matches)
                
                # pagination
                entries_per_page = 10
                pages = utilities.calculate_pages(len(matches)
                    , entries_per_page=entries_per_page)

                # validate the requested page
                current_page = utilities.validate_page_request(
                    request.args.get('page', 1), pages)
                
                # manually paginate
                start_index = entries_per_page * (current_page - 1)
                end_index = start_index + entries_per_page
                paginated_matches = matches[start_index:end_index]

                # list of allowed absolute datetime comparisons
                absolute_comparisons = \
                    constants.filter_comparisons['datetime_absolute']
                
                # if the filter is for a relative datetime, parse the value
                relative_value = None
                if filter.comparison in \
                    constants.filter_comparisons['datetime_relative']:
                    
                    # filter.compare_to is a list [year, month, etc..]
                    # first non zero value will be used
                    for i in range(len(filter.compare_to['value'])):
                        if filter.compare_to['value'][i] != 0:
                            break
                    # values in the compare_to list
                    periods = ['year', 'month', 'week', 'day', 'hour'
                        , 'minute']
                    # see if we need to make it plural
                    suffix = ''
                    if filter.compare_to['value'][i] > 1:
                        suffix = 's'
                    
                    relative_value = '%s %s%s' \
                        % (filter.compare_to['value'][i], periods[i], suffix)

                return render_template('filter.html', filter=filter
                    , entries = paginated_matches
                    , total_matches = len(matches)
                    , project_count=project_count
                    , current_page = current_page
                    , number_of_pages = pages
                    , absolute_datetime_comparisons=absolute_comparisons
                    , relative_datetime_value=relative_value)

        if request.args.get('create', '') == 'true':
            # create a new filter 

            # CSRF validation
            token = request.args.get('token', '')
            if not verify_token(token):
                abort(403)

            try:
                filter_name = 'filter-%s' % utilities.generate_random_string(6)
                new_filter= Filter(
                    creation_time = datetime.datetime.utcnow()
                    , creator = user
                    , label = filter_name.lower()
                    , project = project
                    , name = filter_name
                )
                new_filter.save() 
                app.logger.info('filter created by %s' % session['email'])
                flash('filter created; please change the defaults', 'success')
            except:
                app.logger.error('filter creation failed for %s' % \
                    session['email'])
                flash('There was an error, sorry :/', 'error')
                return redirect(url_for('projects', org_label=org.label
                    , project=project.label))
            
            # redirect to the editing screen
            return redirect(url_for('filters', org_label=org.label
                , project_label=project.label, filter_label=new_filter.label
                , edit='true'))
        
        # no filter in particular was specified, show 'em all
        filters = Filter.objects(project=project)
        
        # list of allowed absolute datetime comparisons
        absolute_comparisons = \
            constants.filter_comparisons['datetime_absolute']
        
        relative_values = {}
        for filter in filters:
            # if the filter is for a relative datetime, parse the value
            relative_value = None
            if filter.comparison in \
                constants.filter_comparisons['datetime_relative']:
                
                # filter.compare_to is a list [year, month, etc..]
                # first non zero value will be used
                for i in range(len(filter.compare_to['value'])):
                    if filter.compare_to['value'][i] != 0:
                        break
                # values in the compare_to list
                periods = ['year', 'month', 'week', 'day', 'hour'
                    , 'minute']
                # see if we need to make it plural
                suffix = ''
                if filter.compare_to['value'][i] > 1:
                    suffix = 's'
                
                relative_value = '%s %s%s' % (
                    filter.compare_to['value'][i], periods[i], suffix)

            relative_values[filter.name] = relative_value

        return render_template('project_filters.html', project=project
            , filters=filters
            , absolute_datetime_comparisons=absolute_comparisons
            , relative_datetime_values=relative_values)
def statistics(org_label, project_label, statistic_label):
    ''' creating statistics for various purposes
    /organizations/aquaya/projects/water-quality/statistics
        : viewing a list of this project's statistics
    /organizations/aquaya/projects/water-quality/statistics?create=true
        : create a new statistic config, immediately redirect to editing
    /organizations/aquaya/projects/water-quality/statistics/mean-pH
        : view a statistic
    /organizations/aquaya/projects/water-quality/statistics/mean-pH?edit=true
        : edit a statistic; accepts GET or POST
    '''
    user = User.objects(email=session['email'])[0]
    
    orgs = Organization.objects(label=org_label)
    if not orgs:
        flash('Organization "%s" not found, sorry!' % org_label, 'warning')
        return redirect(url_for('organizations'))
    org = orgs[0]

    # permission-check
    if org not in user.organizations and not user.admin_rights:
        app.logger.error('%s tried to view a project but was \
            denied for want of admin rights' % session['email'])
        abort(404)
    
    # find the project
    projects = Project.objects(label=project_label, organization=org) 
    if not projects:
        flash('Project "%s" not found, sorry!' % project_label, 'warning')
        return redirect(url_for('organizations', org_label=org.label))
    project = projects[0]

    if request.method == 'POST':
        # we have a statistic_label
        statistics = Statistic.objects(label=statistic_label, project=project)
        if not statistics:
            abort(404)
        statistic = statistics[0]

        form_type = request.form.get('form_type', '')
        if form_type == 'info':
            if statistic.name != request.form.get('name', ''):
                name = request.form.get('name', '')
                statistic.name = name

                statistics = Statistic.objects(project=project).only('label')
                labels = [s.label for s in statistics]
                statistic.label = utilities.generate_label(name, labels)

            statistic.description = request.form.get('description', '')
            statistic.statistic_type = request.form.get('statistic_type', '')

            # header is of the form 'header_id__4abcd0012'
            header = request.form.get('header', '')
            header_id = header.split('header_id__')[1]
            header = Header.objects(id=header_id)[0]
            statistic.header = header


        elif form_type == 'create-table':
            compute_multiple = request.form.get('compute_multiple', '')
            if compute_multiple == 'true':
                statistic.pivot = True
            else:
                statistic.pivot = False

            header_id = request.form.get('compute_multiple_header', '')
            headers = Header.objects(project=project, id=header_id)
            if not headers:
                abort(404)
            statistic.pivot_header = headers[0]
            
            # the name of the select with the values is based on header id
            values = request.form.getlist('header_unique_values')
            escaped_values = [bleach.clean(v) for v in values]
            statistic.pivot_values = escaped_values

            statistic.save()
            
            flash('Table settings saved successfully', 'success')
            return redirect(url_for('statistics'
                , org_label=org.label
                , project_label=project.label
                , statistic_label=statistic.label
                , edit='true'))

        
        elif form_type == 'filters':
            # extract the 'any filters' vs 'all' distinction
            filter_settings = request.form.get('apply_any_filters', '')
            if filter_settings == 'true':
                statistic.apply_any_filters = True
            else:
                statistic.apply_any_filters = False
            statistic.save()
            
            # attach filter to statistic
            requested_filter_ids = request.form.getlist('filters')
            attached_filters = []
            for requested_id in requested_filter_ids:
                prefix, filter_id = requested_id.split('__')
                filters = Filter.objects(id=filter_id)
                if not filters:
                    abort(404)
                attached_filters.append(filters[0])

            statistic.update(set__filters = attached_filters)

            return redirect(url_for('statistics', org_label=org.label
                , project_label=project.label
                , statistic_label=statistic.label, edit='true'))
       

        elif form_type == 'admin':
            # delete the statistic
            name = statistic.name
            utilities.delete_statistic(statistic, session['email'])
            flash('statistic "%s" was deleted successfully' % name, 'success')
            return redirect(url_for('statistics', org_label=org.label
                , project_label=project.label))
        
        else:
            # bad 'form_type'
            abort(404)
       
        try:
            statistic.save()
            flash('Changes saved successfully', 'success')
            return redirect(url_for('statistics', org_label=org.label
                , project_label=project.label
                , statistic_label=statistic.label, edit='true'))
        except:
            app.logger.error('%s experienced an error saving info about %s' % (
                session['email'], request.form['name']))
            flash('Error saving changes -- make sure statistic names are \
                unique.', 'error')
            return redirect(url_for('statistics', org_label=org.label
                , project_label=project.label, statistic_label=statistic_label
                , edit='true'))
        
    
    if request.method == 'GET':
        if statistic_label:
            statistics = Statistic.objects(label=statistic_label
                , project=project)
            if not statistics:
                app.logger.error('%s tried to access a statistic that does \
                    not exist' % session['email'])
                flash('Statistic "%s" not found, sorry!' % statistic_label
                    , 'warning')
                return redirect(url_for('projects', org_label=org.label
                    , project_label=project.label))
            statistic = statistics[0]

            if request.args.get('edit', '') == 'true':
                # valid statistic types
                statistic_types = constants.statistic_types

                available_filters = Filter.objects(project=project)

                # find all string-type headers
                # these are available for compute_multiple
                string_type_headers = []
                for header in project.ordered_schema:
                    if header.display and header.data_type == 'string':
                        string_type_headers.append(header)

                return render_template('statistic_edit.html'
                    , statistic = statistic
                    , statistic_types = statistic_types
                    , available_filters = available_filters
                    , string_type_headers = string_type_headers)
            else:
                # apply the filters and show matches
                conditions = {
                    'project': project
                    , 'unique': True
                    , 'visible': True
                }
                entries = Entry.objects(**conditions)
                
                # sort the entries based on project defaults
                entries = utilities.sort_entries(project, entries)

                project_count = len(entries)
                
                for filter in statistic.filters:
                    if not filter.comparison:
                        flash('filter "%s" does not have a complete \
                            definition' % filter.name, 'warning')
                        return render_template('statistic.html'
                            , statistic=statistic
                            , entries=entries
                            , project_count=project_count)
                
                # apply relevant filters and then computes the statistic
                if statistic.pivot:
                    # returns dict keyed by the statistic's pivot_values
                    result = utilities.compute_pivot_statistic(statistic
                        , entries)
                else:
                    # returns a single result
                    result = utilities.compute_statistic(statistic, entries)

                # find matches to the applied filters
                matches = utilities.apply_filters(statistic.filters, entries
                    , statistic.apply_any_filters)
                total_matches = len(matches)
                
                # pagination
                entries_per_page = 10
                pages = utilities.calculate_pages(len(matches)
                    , entries_per_page=entries_per_page)

                # validate the requested page
                current_page = utilities.validate_page_request(
                    request.args.get('page', 1), pages)

                # manually paginate
                start_index = entries_per_page * (current_page - 1)
                end_index = start_index + entries_per_page
                paginated_matches = matches[start_index:end_index]

                return render_template('statistic.html'
                    , statistic=statistic
                    , entries=paginated_matches
                    , total_matches = total_matches
                    , project_count=project_count
                    , result=result
                    , current_page = current_page
                    , number_of_pages = pages)


        if request.args.get('create', '') == 'true':
            # create a new statistic

            # CSRF validation
            token = request.args.get('token', '')
            if not verify_token(token):
                abort(403)

            try:
                statistic_name = 'stat-%s' \
                    % utilities.generate_random_string(6)
                new_statistic= Statistic(
                    creation_time = datetime.datetime.utcnow()
                    , creator = user
                    , label = statistic_name.lower()
                    , project = project
                    , name = statistic_name
                )
                new_statistic.save() 
                app.logger.info('statistic created by %s' % session['email'])
                flash('statistic created; please change the defaults'
                    , 'success')
            except:
                app.logger.error('statistic creation failed for %s' % \
                    session['email'])
                flash('There was an error, sorry :/', 'error')
                return redirect(url_for('projects', org_label=org.label
                    , project=project.label))
            
            # redirect to the editing screen
            return redirect(url_for('statistics', org_label=org.label
                , project_label=project.label
                , statistic_label=new_statistic.label, edit='true'))
        
        # no statistic in particular was specified; show em all
        statistics = Statistic.objects(project=project)

        # calculate the results of these statistics
        statistic_results = {}
        unique_entries = Entry.objects(project=project, unique=True
            , visible=True)
        for statistic in statistics:
            statistic_results[statistic.name] = \
                utilities.compute_statistic(statistic, unique_entries)

        return render_template('project_statistics.html', project=project
            , statistics=statistics, statistic_results=statistic_results)