def uploads(o_label, p_label, u_label):
''' uploadin
/organizations/aquaya/projects/water-quality/uploads?create=true
: create a new upload, immediately redirect to editing
/organizations/aquaya/projects/water-quality/uploads/may-2012
: view an upload
/organizations/aquaya/projects/water-quality/uploads/may-2012?edit=true
: edit an upload; accepts GET or POST
'''
# can't wrap the line through a route or spaces are injected
# hence this silliness:
org_label = o_label
project_label = p_label
upload_label = u_label
user = User.objects(email=session['email'])[0]
orgs = Organization.objects(label=org_label)
if not orgs:
flash('Organization "%s" not found, sorry!' % org_label, 'warning')
return redirect(url_for('organizations'))
org = orgs[0]
# permission-check
if org not in user.organizations and not user.admin_rights:
app.logger.error('%s tried to view a project but was \
denied for want of admin rights' % session['email'])
abort(404)
projects = Project.objects(label=project_label, organization=org)
if not projects:
flash('Project "%s" not found, sorry!' % project_label, 'warning')
return redirect(url_for('organizations', org_label=org.label))
project = projects[0]
if request.method == 'GET':
if not upload_label and request.args.get('create', '') == 'true':
# create a new upload
# CSRF validation
token = request.args.get('token', '')
if not verify_token(token):
abort(403)
try:
name = 'upl-%s' % utilities.generate_random_string(6)
new_upload = Upload(
label = name.lower()
, name = name
, project = project
)
new_upload.save()
project.update(push__uploads=new_upload)
app.logger.info('upload created by %s' % session['email'])
flash('please select a file and make other edits', 'warning')
except:
app.logger.error('upload creation failed for %s' % \
session['email'])
flash('There was an error, sorry :/', 'error')
return redirect(url_for('projects', org_label=org.label
, project_label=project.label))
# redirect to the editing screen
return redirect(url_for('uploads', o_label=org.label
, p_label=project.label, u_label=new_upload.label
, edit='true'))
elif not upload_label:
# could show projects uploads but punting for now
unique_entries = Entry.objects(project=project, unique=True
, visible=True)
duplicate_entries = Entry.objects(project=project
, unique=False)
hidden_entries = Entry.objects(project=project, visible=False
, unique=True)
upload_entry_counts = {}
for upload in project.uploads:
upload_entry_counts[upload.id] = Entry.objects(upload=upload).count()
return render_template('project_uploads.html', project=project
, unique_entries=unique_entries
, duplicate_entries=duplicate_entries
, upload_entry_counts=upload_entry_counts
, hidden_entries=hidden_entries)
else:
# we have an upload label
uploads = Upload.objects(label=upload_label, project=project)
if not uploads:
flash('Upload "%s" not found, sorry!' % upload_label
, 'warning')
return redirect(url_for('projects', org_label=org.label
, project_label=project.label))
upload = uploads[0]
if request.args.get('edit', '') == 'true':
# show the editor
# queue up any flashed messages
# these get set during the worker's processing of a file
worker_messages = upload.worker_messages
for message in worker_messages:
flash(message['message'], message['status'])
upload.update(set__worker_messages = [])
return render_template('upload_edit.html', upload=upload
, number_of_pages=0)
else:
if not upload.filename:
# need to add one so redirect to editing
flash('please specify a file for this upload', 'warning')
return redirect(url_for('uploads', o_label=org.label
, p_label=project.label, u_label=upload.label
, edit='true'))
else:
# things are still in progress
if not upload.s3_key:
flash('Your file is being processed in the '
'background. Refresh the page to see updates.'
, 'info')
return render_template('upload.html', upload=upload
, number_of_pages=0)
# otherwise upload is complete, show the data
# pagination
total_entries = Entry.objects(project=project
, upload=upload).count()
entries_per_page = 10
pages = utilities.calculate_pages(total_entries
, entries_per_page=entries_per_page)
# validate the requested page
current_page = utilities.validate_page_request(
request.args.get('page', 1), pages)
# get the sorted entries
entries = utilities.query_entries(project
, upload = upload
, skip = (entries_per_page * (current_page - 1))
, limit = entries_per_page)
hidden_entries_count = Entry.objects(upload=upload
, visible=False).count()
# queue up any flashed messages
# these get set during the worker's processing of a file
worker_messages = upload.worker_messages
for message in worker_messages:
flash(message['message'], message['status'])
upload.update(set__worker_messages = [])
return render_template('upload.html'
, upload=upload
, entries=entries
, total_entries = total_entries
, hidden_entries_count=hidden_entries_count
, current_page = current_page
, number_of_pages = pages)
elif request.method == 'POST':
# we have an upload label
uploads = Upload.objects(label=upload_label, project=project)
if not uploads:
abort(404)
upload = uploads[0]
form_type = request.form.get('form_type', '')
if form_type == 'info':
if upload.name != request.form.get('name', ''):
name = request.form.get('name', '')
upload.update(set__name = name)
uploads = Upload.objects(project=project).only('label')
labels = [u.label for u in uploads]
upload.update(set__label =
utilities.generate_label(name, labels))
upload.project.update(set__update_time =
datetime.datetime.utcnow())
upload.reload()
if upload.description != request.form.get('description', ''):
upload.update(set__description =
request.form.get('description', ''))
upload.project.update(set__update_time =
datetime.datetime.utcnow())
if request.files.get('data_file', ''):
data_file = request.files.get('data_file')
try:
filename = uploaded_data.save(data_file)
if '..' in filename or filename.startswith('/'):
app.logger.info('%s tried to upload a malicious file \
"%s"' % (session['email'], filename))
flash('bad filename, sorry :/', 'error')
return redirect(url_for('uploads'
, o_label=org.label, p_label=project.label
, u_label=upload.label, edit='true'))
absolute_filename = uploaded_data.path(filename)
except UploadNotAllowed:
app.logger.info('%s tried to upload an unsupported file'
% session['email'])
flash('we currently only support .xls files, sorry :/'
, 'error')
return redirect(url_for('uploads'
, o_label=org.label, p_label=project.label
, u_label=upload.label, edit='true'))
upload.update(set__filename =
os.path.basename(absolute_filename))
upload.update(set__uploaded_by = user)
# enqueue upload-processing
redis_config = app.config['REDIS_CONFIG']
use_connection(Redis(redis_config['host'], redis_config['port']
, password=redis_config['password']))
queue = Queue()
queue.enqueue(process_uploaded_file, upload.id,
absolute_filename)
elif form_type == 'admin':
# delete the upload
name = upload.name
# pull out the upload from the parent project first
project.update(pull__uploads=upload)
# delete associated entries, remove from system, remove from s3
deleted_entries_count = utilities.delete_upload(upload
, session['email'])
flash('upload "%s" was deleted successfully; %s entries deleted \
as well' % (name, deleted_entries_count), 'success')
# see if, given this deletion, the dupes are still dupes
unduped_count = utilities.recheck_duplicate_entries(project)
if unduped_count:
flash('%s entries were automatically added to the project \
as they are no longer duplicates' % unduped_count, 'info')
return redirect(url_for('uploads', o_label=org.label
, p_label=project.label))
else:
# bad 'form_type'
abort(404)
try:
upload.save()
project.save()
except:
app.logger.error('%s experienced an error saving info about the \
upload %s' % (session['email'], request.form['name']))
flash('error, make sure names are unique', 'error')
return redirect(url_for('projects'
, org_label=upload.project.organization.label
, project_label=upload.project.label))
return redirect(url_for('uploads'
, o_label=upload.project.organization.label, p_label=project.label
, u_label=upload.label))
def entries(org_label, project_label, entry_id):
''' show a specific entry mostly for editing purposes
'''
user = User.objects(email=session['email'])[0]
# find the org
orgs = Organization.objects(label=org_label)
if not orgs:
flash('Organization "%s" not found, sorry!' % org_label, 'warning')
return redirect(url_for('organizations'))
org = orgs[0]
# permission-check
if org not in user.organizations and not user.admin_rights:
app.logger.error('%s tried to view a project but was \
denied for want of admin rights' % session['email'])
abort(404)
# find the project
projects = Project.objects(label=project_label, organization=org)
if not projects:
flash('Project "%s" not found, sorry!' % project_label, 'warning')
return redirect(url_for('organizations', org_label=org.label))
project = projects[0]
if not entry_id:
if request.method == 'POST':
# downloading entries with applied filters
filter_labels = request.form.getlist('filters')
apply_any_filters = request.form.get('apply_any_filters', '')
if apply_any_filters == 'true':
apply_any_filters = True
else:
apply_any_filters = False
# make a list of filter objects
filters = []
for filter_label in filter_labels:
filters.append(Filter.objects(label=filter_label
, project=project)[0])
# serve up a file of the project entries
absolute_filename = utilities.download_all_entries(project
, filters, format='xls', apply_any_filters=apply_any_filters)
# delay the deletion so we have time to serve the file
redis_config = app.config['REDIS_CONFIG']
use_connection(Redis(redis_config['host'], redis_config['port']
, password=redis_config['password']))
scheduler = Scheduler()
scheduler.enqueue_in(datetime.timedelta(seconds=60)
, delete_local_file, absolute_filename)
return send_file(absolute_filename, as_attachment=True)
'''GET request..
display entries for the project
'''
duplicate_entries_count = Entry.objects(project=project
, unique=False).count()
hidden_entries_count = Entry.objects(project=project, unique=True
, visible=False).count()
unique_entries_count = Entry.objects(project=project, unique=True
, visible=True).count()
entry_type = request.args.get('type', '')
if entry_type == 'duplicate':
# show unique=False entries
count = duplicate_entries_count
unique = False
visible = None
template = 'project_duplicate_entries.html'
elif entry_type == 'hidden':
# show visible=False, unique=True entries
count = hidden_entries_count
unique = True
visible = False
template = 'project_hidden_entries.html'
else:
# show uniques
count = unique_entries_count
unique = True
visible = True
template = 'project_entries.html'
entries_per_page = 10
pages = utilities.calculate_pages(count
, entries_per_page=entries_per_page)
# validate the requested page
current_page = utilities.validate_page_request(
request.args.get('page', 1), pages)
entries = utilities.query_entries(project
, unique=unique
, visible=visible
, skip=(entries_per_page * (current_page - 1))
, limit=entries_per_page)
# present some filters if data is downloaded
available_filters = Filter.objects(project=project)
return render_template(template
, project=project
, entries=entries
, unique_entries_count=unique_entries_count
, duplicate_entries_count=duplicate_entries_count
, hidden_entries_count=hidden_entries_count
, available_filters=available_filters
, current_page = current_page
, number_of_pages = pages)
# we have an entry_id, try to find the object
entries = Entry.objects(id=entry_id)
if not entries:
flash('Entry "%s" not found, sorry!' % entry_id, 'warning')
return redirect(url_for('entries', org_label=org.label
, project_label=project.label))
entry = entries[0]
if request.method == 'GET':
if request.args.get('edit', '') == 'true':
return render_template('entry_edit.html', entry=entry)
else:
return render_template('entry.html', entry=entry)
elif request.method == 'POST':
form_type = request.form.get('form_type', '')
if form_type == 'info':
# track all modifications to this entry
modifications = []
# don't think I can set just one value in the dict with set__
# so let's make a copy then alter it, then update it
values = dict(entry.values)
for header in entry.project.ordered_schema:
if header.data_type == 'datetime':
full_dt = '%s %s' % (
request.form.get('%s__date' % header.id, '')
, request.form.get('%s__time' % header.id, ''))
try:
struct = time.strptime(full_dt, '%B %d, %Y %I:%M %p')
edited_val = datetime.datetime.fromtimestamp(
time.mktime(struct))
except ValueError:
# requested change was improperly formatted
message = ('Error. Date-time data expected for the'
' field "%s."' % header.label)
return redirect_to_editing(entry, 'error', message)
elif header.data_type == 'number':
try:
edited_val = float(request.form.get(
str(header.id), ''))
except ValueError:
# requested change wasn't a number
message = ('Error. Numerical data expected for the'
' field "%s."' % header.label)
return redirect_to_editing(entry, 'error', message)
else:
edited_val = request.form.get(str(header.id), '')
# values that were originally None will show up here as ''
if edited_val == '' and entry.values[header.name] == None:
continue
if edited_val != entry.values[header.name]:
values[header.name] = edited_val
modifications.append('updated "%s" from "%s" to "%s"' %
(header.label, entry.values[header.name]
, edited_val))
if modifications:
''' update the entry
'''
entry.update(set__values = values)
entry.update(set__was_never_edited = False)
''' generate some hashes
used to check other possible duplicative and hidden entries
'''
old_hash = str(entry.value_hash)
# compute a hash for the new values
m = hashlib.md5()
#sorted_headers = [h.name for h in project.ordered_schema]
sorted_headers = values.keys()
sorted_headers.sort()
for header in sorted_headers:
value = values[header]
if type(value) == type(u'unicode'):
m.update(value.encode('utf8'))
else:
m.update(str(value))
new_hash = m.hexdigest()
# if the old entry was unique and had a dupe..
# ..that dupe is now unique
# limit one to flip just one of several possible dupes
if entry.unique:
old_dupes = Entry.objects(project=project, unique=False
, value_hash=old_hash).limit(1)
if old_dupes:
flash('The entry you have edited had a duplicate in'
' the system. That duplicate has now been marked'
' "unique."', 'success')
old_dupes[0].update(set__unique = True)
# process hidden entries
# if there are hidden entries with these values..
# ..sound the alarm
hidden_entries = Entry.objects(project=project
, visible=False).only('value_hash')
hidden_hashes = [h['value_hash'] for h in hidden_entries]
if entry.visible and new_hash in hidden_hashes:
flash('Warning: an entry with these values was previously'
' "hidden," i.e. removed from analysis. Consider'
' hiding this entry.', 'warning')
# if the entry was hidden, remind the user of that fact..
# since the values were just edited they may want to change it
if not entry.visible:
flash('This entry is currently "hidden" and not included'
' in analysis. Consider un-hiding it to include your'
' new edits.', 'warning')
# search for unique and duplicate values
uniques = Entry.objects(project=project
, unique=True).only('value_hash')
unique_hashes = [u['value_hash'] for u in uniques]
# if entry /was/ unique
if entry.unique:
if new_hash in unique_hashes:
flash('This entry is now a duplicate of another entry'
' in the system.', 'warning')
entry.update(set__unique=False)
# entry /wasn't/ unique
else:
if new_hash not in unique_hashes:
flash('This entry was formerly a duplicate but is now'
' unique.', 'success')
entry.update(set__unique=True)
entry.update(set__value_hash = new_hash)
''' create a comment encapsulating the changes
'''
new_comment = Comment(
body = '; '.join(modifications)
, creation_time = datetime.datetime.utcnow()
, editable = False
, entry = entry
, owner = user
, project = project
)
new_comment.save()
message = ('Changes saved successfully: %s' %
'; '.join(modifications))
return redirect_to_editing(entry, 'success', message)
else:
# no modifications made to the entry
return redirect_to_editing(entry, None, None)
elif form_type == 'hide_entry':
# flip the 'visible' state of this entry
if entry.visible:
entry.update(set__visible = False)
modifications = 'entry removed from analysis'
else:
entry.update(set__visible = True)
modifications = 'entry re-included in analysis'
# also find all duplicates in the project and hide/unhide them as well
duplicate_entries = Entry.objects(project=project
, value_hash=entry.value_hash)
# if we have more than the original entry
if len(duplicate_entries) > 1:
for duplicate_entry in duplicate_entries:
if duplicate_entry.id == entry.id:
continue
entry.reload()
if entry.visible:
duplicate_entry.update(set__visible = True)
else:
duplicate_entry.update(set__visible = False)
# append a note about the dupes
# plural..
dupes = len(duplicate_entries) - 1
if dupes > 1:
plural = 's'
else:
plural = ''
modifications += ' with %s duplicate%s' % (dupes, plural)
''' create a comment encapsulating the changes
hm, comments won't be attached to duplicates..alas
'''
new_comment = Comment(
body = modifications
, creation_time = datetime.datetime.utcnow()
, editable = False
, entry = entry
, owner = user
, project = project
)
new_comment.save()
message = 'Changes saved successfully: %s.' % modifications
return redirect_to_editing(entry, 'success', message)
def filters(org_label, project_label, filter_label):
''' creating filters for various purposes
/organizations/aquaya/projects/water-quality/filters?create=true
: create a new filter config, immediately redirect to editing
/organizations/aquaya/projects/water-quality/filters/big-cities
: view a filter
/organizations/aquaya/projects/water-quality/filters/big-cities?edit=true
: edit a filter; accepts GET or POST
'''
user = User.objects(email=session['email'])[0]
orgs = Organization.objects(label=org_label)
if not orgs:
flash('Organization "%s" not found, sorry!' % org_label, 'warning')
return redirect(url_for('organizations'))
org = orgs[0]
# permission-check
if org not in user.organizations and not user.admin_rights:
app.logger.error('%s tried to view a project but was \
denied for want of admin rights' % session['email'])
abort(404)
# find the project
projects = Project.objects(label=project_label, organization=org)
if not projects:
flash('Project "%s" not found, sorry!' % project_label, 'warning')
return redirect(url_for('organizations', org_label=org.label))
project = projects[0]
if request.method == 'POST':
# we have a filter_label
filters= Filter.objects(label=filter_label, project=project)
if not filters:
abort(404)
# should try to avoid overriding the builtin..
filter = filters[0]
form_type = request.form.get('form_type', '')
if form_type == 'info':
if filter.name != request.form.get('name', ''):
name = request.form.get('name', '')
filter.name = name
filters = Filter.objects(project=project).only('label')
labels = [f.label for f in filters]
filter.label = utilities.generate_label(name, labels)
filter.description = request.form.get('description', '')
filter.comparison = request.form.get('comparison', '')
# header of the format header_id__4abcd0001
header_id = request.form.get('header', '')
header_id = header_id.split('header_id__')[1]
header = Header.objects(id=header_id)[0]
filter.header = header
if header.data_type == 'datetime':
# check if relative or absolute datetime comparison
if filter.comparison in \
constants.filter_comparisons['datetime_absolute']:
compare_to = '%s %s' % (
request.form.get('date_compare_to', '')
, request.form.get('time_compare_to', ''))
try:
dt = datetime.datetime.strptime(
compare_to, '%B %d, %Y %I:%M %p')
filter.compare_to = {'value': dt}
except:
flash("didn't understand that date formatting; make \
sure it looks like 'June 21, 2012' and \
'02:45 PM'", 'error')
return redirect(url_for('filters', org_label=org.label
, project_label=project.label
, filter_label=filter.label
, edit='true'))
elif filter.comparison in \
constants.filter_comparisons['datetime_relative']:
inputs = [request.form.get('relative_years', '')
, request.form.get('relative_months', '')
, request.form.get('relative_weeks', '')
, request.form.get('relative_days', '')
, request.form.get('relative_hours', '')
, request.form.get('relative_minutes', '')]
parsed_inputs = []
for i in inputs:
if i:
parsed_inputs.append(i)
else:
parsed_inputs.append(0)
# verify that we have numbers
try:
values = [int(i) for i in parsed_inputs]
except:
flash('input times must be whole numbers', 'error')
return redirect(url_for('filters', org_label=org.label
, project_label=project.label
, filter_label=filter.label
, edit='true'))
filter.compare_to = {'value': values}
else:
app.logger.info('unknown comparison "%s" for project %s' \
% (comparison, project.name))
abort(404)
else:
# not datetime; cast the value we're comparing against
compare_to = request.form.get('string_compare_to', '')
filter.compare_to = {'value': utilities.smart_cast(compare_to)}
elif form_type == 'admin':
# delete the filter
name = filter.name
# pull filters out of the statistics
statistics = Statistic.objects(filters=filter)
for statistic in statistics:
statistic.update(pull__filters=filter)
# pull filters out of the graphs
graphs = Graph.objects(filters=filter)
for graph in graphs:
graph.update(pull__filters=filter)
filter.delete()
app.logger.info('%s deleted filter "%s"' % \
(session['email'], name))
flash('filter "%s" was deleted successfully' % name, 'success')
return redirect(url_for('filters', org_label=org.label
, project_label=project.label))
else:
# bad 'form_type'
abort(404)
try:
filter.save()
flash('changes saved successfully', 'success')
return redirect(url_for('filters', org_label=org.label
, project_label=project.label, filter_label=filter.label))
except:
app.logger.error('%s experienced an error saving info about %s' % (
session['email'], request.form['name']))
flash('Error saving changes -- make sure filter names are unique.'
, 'error')
return redirect(url_for('filters', org_label=org.label
, project_label=project.label, filter_label=filter_label
, edit='true'))
if request.method == 'GET':
if filter_label:
filters = Filter.objects(label=filter_label, project=project)
if not filters:
app.logger.error('%s tried to access a filter that does not \
exist' % session['email'])
flash('Filter "%s" not found, sorry!' % filter_label
, 'warning')
return redirect(url_for('projects', org_label=org.label
, project_label=project.label))
filter = filters[0]
if request.args.get('edit', '') == 'true':
# valid comparisons
comparisons = json.dumps(constants.filter_comparisons)
# list of allowed absolute datetime comparisons
absolute_comparisons = \
constants.filter_comparisons['datetime_absolute']
relative_values = None
if filter.comparison in \
constants.filter_comparisons['datetime_relative']:
relative_values = filter.compare_to['value']
return render_template('filter_edit.html', filter=filter
, comparisons=comparisons
, absolute_datetime_comparisons=absolute_comparisons
, relative_datetime_compare_values=relative_values)
else:
# apply the filter; start with some defaults
conditions = {
'project': project
, 'unique': True
, 'visible': True
}
entries = Entry.objects(**conditions)
project_count = len(entries)
if not filter.comparison:
return render_template('filter.html', filter=filter
, project_count=project_count)
# find all matches for this filter to display
matches = utilities.apply_filter(filter, entries)
# sort the matches
matches = utilities.sort_entries(project, matches)
# pagination
entries_per_page = 10
pages = utilities.calculate_pages(len(matches)
, entries_per_page=entries_per_page)
# validate the requested page
current_page = utilities.validate_page_request(
request.args.get('page', 1), pages)
# manually paginate
start_index = entries_per_page * (current_page - 1)
end_index = start_index + entries_per_page
paginated_matches = matches[start_index:end_index]
# list of allowed absolute datetime comparisons
absolute_comparisons = \
constants.filter_comparisons['datetime_absolute']
# if the filter is for a relative datetime, parse the value
relative_value = None
if filter.comparison in \
constants.filter_comparisons['datetime_relative']:
# filter.compare_to is a list [year, month, etc..]
# first non zero value will be used
for i in range(len(filter.compare_to['value'])):
if filter.compare_to['value'][i] != 0:
break
# values in the compare_to list
periods = ['year', 'month', 'week', 'day', 'hour'
, 'minute']
# see if we need to make it plural
suffix = ''
if filter.compare_to['value'][i] > 1:
suffix = 's'
relative_value = '%s %s%s' \
% (filter.compare_to['value'][i], periods[i], suffix)
return render_template('filter.html', filter=filter
, entries = paginated_matches
, total_matches = len(matches)
, project_count=project_count
, current_page = current_page
, number_of_pages = pages
, absolute_datetime_comparisons=absolute_comparisons
, relative_datetime_value=relative_value)
if request.args.get('create', '') == 'true':
# create a new filter
# CSRF validation
token = request.args.get('token', '')
if not verify_token(token):
abort(403)
try:
filter_name = 'filter-%s' % utilities.generate_random_string(6)
new_filter= Filter(
creation_time = datetime.datetime.utcnow()
, creator = user
, label = filter_name.lower()
, project = project
, name = filter_name
)
new_filter.save()
app.logger.info('filter created by %s' % session['email'])
flash('filter created; please change the defaults', 'success')
except:
app.logger.error('filter creation failed for %s' % \
session['email'])
flash('There was an error, sorry :/', 'error')
return redirect(url_for('projects', org_label=org.label
, project=project.label))
# redirect to the editing screen
return redirect(url_for('filters', org_label=org.label
, project_label=project.label, filter_label=new_filter.label
, edit='true'))
# no filter in particular was specified, show 'em all
filters = Filter.objects(project=project)
# list of allowed absolute datetime comparisons
absolute_comparisons = \
constants.filter_comparisons['datetime_absolute']
relative_values = {}
for filter in filters:
# if the filter is for a relative datetime, parse the value
relative_value = None
if filter.comparison in \
constants.filter_comparisons['datetime_relative']:
# filter.compare_to is a list [year, month, etc..]
# first non zero value will be used
for i in range(len(filter.compare_to['value'])):
if filter.compare_to['value'][i] != 0:
break
# values in the compare_to list
periods = ['year', 'month', 'week', 'day', 'hour'
, 'minute']
# see if we need to make it plural
suffix = ''
if filter.compare_to['value'][i] > 1:
suffix = 's'
relative_value = '%s %s%s' % (
filter.compare_to['value'][i], periods[i], suffix)
relative_values[filter.name] = relative_value
return render_template('project_filters.html', project=project
, filters=filters
, absolute_datetime_comparisons=absolute_comparisons
, relative_datetime_values=relative_values)
def statistics(org_label, project_label, statistic_label):
''' creating statistics for various purposes
/organizations/aquaya/projects/water-quality/statistics
: viewing a list of this project's statistics
/organizations/aquaya/projects/water-quality/statistics?create=true
: create a new statistic config, immediately redirect to editing
/organizations/aquaya/projects/water-quality/statistics/mean-pH
: view a statistic
/organizations/aquaya/projects/water-quality/statistics/mean-pH?edit=true
: edit a statistic; accepts GET or POST
'''
user = User.objects(email=session['email'])[0]
orgs = Organization.objects(label=org_label)
if not orgs:
flash('Organization "%s" not found, sorry!' % org_label, 'warning')
return redirect(url_for('organizations'))
org = orgs[0]
# permission-check
if org not in user.organizations and not user.admin_rights:
app.logger.error('%s tried to view a project but was \
denied for want of admin rights' % session['email'])
abort(404)
# find the project
projects = Project.objects(label=project_label, organization=org)
if not projects:
flash('Project "%s" not found, sorry!' % project_label, 'warning')
return redirect(url_for('organizations', org_label=org.label))
project = projects[0]
if request.method == 'POST':
# we have a statistic_label
statistics = Statistic.objects(label=statistic_label, project=project)
if not statistics:
abort(404)
statistic = statistics[0]
form_type = request.form.get('form_type', '')
if form_type == 'info':
if statistic.name != request.form.get('name', ''):
name = request.form.get('name', '')
statistic.name = name
statistics = Statistic.objects(project=project).only('label')
labels = [s.label for s in statistics]
statistic.label = utilities.generate_label(name, labels)
statistic.description = request.form.get('description', '')
statistic.statistic_type = request.form.get('statistic_type', '')
# header is of the form 'header_id__4abcd0012'
header = request.form.get('header', '')
header_id = header.split('header_id__')[1]
header = Header.objects(id=header_id)[0]
statistic.header = header
elif form_type == 'create-table':
compute_multiple = request.form.get('compute_multiple', '')
if compute_multiple == 'true':
statistic.pivot = True
else:
statistic.pivot = False
header_id = request.form.get('compute_multiple_header', '')
headers = Header.objects(project=project, id=header_id)
if not headers:
abort(404)
statistic.pivot_header = headers[0]
# the name of the select with the values is based on header id
values = request.form.getlist('header_unique_values')
escaped_values = [bleach.clean(v) for v in values]
statistic.pivot_values = escaped_values
statistic.save()
flash('Table settings saved successfully', 'success')
return redirect(url_for('statistics'
, org_label=org.label
, project_label=project.label
, statistic_label=statistic.label
, edit='true'))
elif form_type == 'filters':
# extract the 'any filters' vs 'all' distinction
filter_settings = request.form.get('apply_any_filters', '')
if filter_settings == 'true':
statistic.apply_any_filters = True
else:
statistic.apply_any_filters = False
statistic.save()
# attach filter to statistic
requested_filter_ids = request.form.getlist('filters')
attached_filters = []
for requested_id in requested_filter_ids:
prefix, filter_id = requested_id.split('__')
filters = Filter.objects(id=filter_id)
if not filters:
abort(404)
attached_filters.append(filters[0])
statistic.update(set__filters = attached_filters)
return redirect(url_for('statistics', org_label=org.label
, project_label=project.label
, statistic_label=statistic.label, edit='true'))
elif form_type == 'admin':
# delete the statistic
name = statistic.name
utilities.delete_statistic(statistic, session['email'])
flash('statistic "%s" was deleted successfully' % name, 'success')
return redirect(url_for('statistics', org_label=org.label
, project_label=project.label))
else:
# bad 'form_type'
abort(404)
try:
statistic.save()
flash('Changes saved successfully', 'success')
return redirect(url_for('statistics', org_label=org.label
, project_label=project.label
, statistic_label=statistic.label, edit='true'))
except:
app.logger.error('%s experienced an error saving info about %s' % (
session['email'], request.form['name']))
flash('Error saving changes -- make sure statistic names are \
unique.', 'error')
return redirect(url_for('statistics', org_label=org.label
, project_label=project.label, statistic_label=statistic_label
, edit='true'))
if request.method == 'GET':
if statistic_label:
statistics = Statistic.objects(label=statistic_label
, project=project)
if not statistics:
app.logger.error('%s tried to access a statistic that does \
not exist' % session['email'])
flash('Statistic "%s" not found, sorry!' % statistic_label
, 'warning')
return redirect(url_for('projects', org_label=org.label
, project_label=project.label))
statistic = statistics[0]
if request.args.get('edit', '') == 'true':
# valid statistic types
statistic_types = constants.statistic_types
available_filters = Filter.objects(project=project)
# find all string-type headers
# these are available for compute_multiple
string_type_headers = []
for header in project.ordered_schema:
if header.display and header.data_type == 'string':
string_type_headers.append(header)
return render_template('statistic_edit.html'
, statistic = statistic
, statistic_types = statistic_types
, available_filters = available_filters
, string_type_headers = string_type_headers)
else:
# apply the filters and show matches
conditions = {
'project': project
, 'unique': True
, 'visible': True
}
entries = Entry.objects(**conditions)
# sort the entries based on project defaults
entries = utilities.sort_entries(project, entries)
project_count = len(entries)
for filter in statistic.filters:
if not filter.comparison:
flash('filter "%s" does not have a complete \
definition' % filter.name, 'warning')
return render_template('statistic.html'
, statistic=statistic
, entries=entries
, project_count=project_count)
# apply relevant filters and then computes the statistic
if statistic.pivot:
# returns dict keyed by the statistic's pivot_values
result = utilities.compute_pivot_statistic(statistic
, entries)
else:
# returns a single result
result = utilities.compute_statistic(statistic, entries)
# find matches to the applied filters
matches = utilities.apply_filters(statistic.filters, entries
, statistic.apply_any_filters)
total_matches = len(matches)
# pagination
entries_per_page = 10
pages = utilities.calculate_pages(len(matches)
, entries_per_page=entries_per_page)
# validate the requested page
current_page = utilities.validate_page_request(
request.args.get('page', 1), pages)
# manually paginate
start_index = entries_per_page * (current_page - 1)
end_index = start_index + entries_per_page
paginated_matches = matches[start_index:end_index]
return render_template('statistic.html'
, statistic=statistic
, entries=paginated_matches
, total_matches = total_matches
, project_count=project_count
, result=result
, current_page = current_page
, number_of_pages = pages)
if request.args.get('create', '') == 'true':
# create a new statistic
# CSRF validation
token = request.args.get('token', '')
if not verify_token(token):
abort(403)
try:
statistic_name = 'stat-%s' \
% utilities.generate_random_string(6)
new_statistic= Statistic(
creation_time = datetime.datetime.utcnow()
, creator = user
, label = statistic_name.lower()
, project = project
, name = statistic_name
)
new_statistic.save()
app.logger.info('statistic created by %s' % session['email'])
flash('statistic created; please change the defaults'
, 'success')
except:
app.logger.error('statistic creation failed for %s' % \
session['email'])
flash('There was an error, sorry :/', 'error')
return redirect(url_for('projects', org_label=org.label
, project=project.label))
# redirect to the editing screen
return redirect(url_for('statistics', org_label=org.label
, project_label=project.label
, statistic_label=new_statistic.label, edit='true'))
# no statistic in particular was specified; show em all
statistics = Statistic.objects(project=project)
# calculate the results of these statistics
statistic_results = {}
unique_entries = Entry.objects(project=project, unique=True
, visible=True)
for statistic in statistics:
statistic_results[statistic.name] = \
utilities.compute_statistic(statistic, unique_entries)
return render_template('project_statistics.html', project=project
, statistics=statistics, statistic_results=statistic_results)