def visitIfExpFlat(self, n, target):
# Setup Values
testVal = self.dispatch(n.test)
thenVal = self.dispatch(n.then.expr)
elseVal = self.dispatch(n.else_.expr)
# Setup Labels
testL = LabelArgLLVM(LocalLLVM(generate_label("test")))
thenL = LabelArgLLVM(LocalLLVM(generate_label("then")))
elseL = LabelArgLLVM(LocalLLVM(generate_label("else")))
endL = LabelArgLLVM(LocalLLVM(generate_label("end")))
# Test Block
thenS = SwitchPairLLVM(DEFAULTTRUE, thenL)
testI = []
testI += [switchLLVM(testVal, elseL, [thenS])]
testB = [blockLLVM(testL, testI)]
# Then Block
thenB = self.dispatch(n.then.node, None)
# Patch in proper label for first block
thenB[0].label = thenL
# Patch in proper destination in last block
thenB[-1].instrs[-1].defaultDest = endL
# Else Block
elseB = self.dispatch(n.else_.node, None)
# Patch in proper label for first block
elseB[0].label = elseL
# Patch in proper destination in last block
elseB[-1].instrs[-1].defaultDest = endL
# End Block
thenP = PhiPairLLVM(thenVal, thenB[-1].label)
elseP = PhiPairLLVM(elseVal, elseB[-1].label)
endI = []
endI += [phiLLVM(target, [thenP, elseP])]
endI += [switchLLVM(DEFAULTZERO, DUMMYL, [])]
endB = [blockLLVM(endL, endI)]
return testB + thenB + elseB + endB
def visitIfPhi(self, n, func_name):
blocks = []
phis = {}
# Setup Labels
nextL = LabelArgLLVM(LocalLLVM(generate_label("if")))
endL = LabelArgLLVM(LocalLLVM(generate_label("end")))
# Tests
for test in n.tests:
(test, tbody, phiD) = test
testV = self.dispatch(test)
testB = self.dispatch(tbody, func_name)
testL = nextL
nextL = LabelArgLLVM(LocalLLVM(generate_label("if")))
testS = SwitchPairLLVM(DEFAULTTRUE, testB[0].label)
testI = [switchLLVM(testV, nextL, [testS])]
for key in phiD.keys():
phiV = VarLLVM(LocalLLVM(phiD[key]), DEFAULTTYPE)
phiP = [PhiPairLLVM(phiV, testB[-1].label)]
if key in phis:
phis[key] += phiP
else:
phis[key] = phiP
testB[-1].instrs[-1].defaultDest = endL
blocks += [blockLLVM(testL, testI)] + testB
# Else
(ebody, phiD) = n.else_
elseB = self.dispatch(ebody, func_name)
elseL = nextL
nextL = LabelArgLLVM(LocalLLVM(generate_label("if")))
elseI = [switchLLVM(DEFAULTZERO, elseB[0].label, [])]
for key in phiD.keys():
phiV = VarLLVM(LocalLLVM(phiD[key]), DEFAULTTYPE)
phiP = [PhiPairLLVM(phiV, elseB[-1].label)]
if key in phis:
phis[key] += phiP
else:
phis[key] = phiP
elseB[-1].instrs[-1].defaultDest = endL
blocks += [blockLLVM(elseL, elseI)] + elseB
# End
endI = []
for key in phis.keys():
phiT = VarLLVM(LocalLLVM(key), DEFAULTTYPE)
endI += [phiLLVM(phiT, phis[key])]
endI += [switchLLVM(DEFAULTZERO, DUMMYL, [])]
blocks += [blockLLVM(endL, endI)]
return blocks
def visitStmtList(self, n, func):
instrs = []
blocks = []
thisL = LabelArgLLVM(LocalLLVM(generate_label("block")))
nextL = LabelArgLLVM(LocalLLVM(generate_label("block")))
length = len(n.nodes)
if(length > 0):
# None-Empty List
for node in n.nodes:
ret = self.dispatch(node, func)
if(isinstance(ret[0], BlockLLVMInst)):
# If list of blocks returned
# Add jump to start of first returned block
instrs += [switchLLVM(DEFAULTZERO, ret[0].label, [])]
# Add new block
blocks += [blockLLVM(thisL, instrs)]
# Patch in proper destination in last block
ret[-1].instrs[-1].defaultDest = nextL
# Add returned blocks
blocks += ret
# Reset instructiosn and update labels
instrs = []
thisL = nextL
nextL = LabelArgLLVM(LocalLLVM(generate_label("block")))
elif(isinstance(ret[0], LLVMInst)):
# If list of instructions returned
instrs += ret
if((node is n.nodes[-1]) and not(isinstance(instrs[-1], TermLLVMInst))):
# If this is the last node and no terminal instruction has been reached
instrs += [switchLLVM(DEFAULTZERO, DUMMYL, [])]
if(isinstance(instrs[-1], TermLLVMInst)):
# If last instruction returned is terminal, end block
blocks += [blockLLVM(thisL, instrs)]
instrs = []
thisL = nextL
nextL = LabelArgLLVM(LocalLLVM(generate_label("block")))
else:
raise Exception("Unhandled return type")
else:
# Empty List
# Add Dummy Block (will be patched into something usefull later)
blocks += [blockLLVM(DUMMYL, [switchLLVM(DEFAULTZERO, DUMMYL, [])])]
return blocks
def visitWhileFlatPhi(self, n, func):
# Setup Values
testVal = self.dispatch(n.test)
# Setup Labels
entryL = LabelArgLLVM(LocalLLVM(generate_label("wentry")))
testprepL = LabelArgLLVM(LocalLLVM(generate_label("wtestprep")))
testL = LabelArgLLVM(LocalLLVM(generate_label("wtest")))
loopL = LabelArgLLVM(LocalLLVM(generate_label("wloop")))
elseL = LabelArgLLVM(LocalLLVM(generate_label("welse")))
endL = LabelArgLLVM(LocalLLVM(generate_label("wend")))
# Loop Block
loopB = self.dispatch(n.body, None)
# Patch in proper label for first block
loopB[0].label = loopL
# Patch in proper destination in last block
loopB[-1].instrs[-1].defaultDest = testprepL
# Entry Block
entryI = []
entryI += [switchLLVM(DEFAULTZERO, testprepL, [])]
entryB = [blockLLVM(entryL, entryI)]
# TestPrep Block
phis = []
for key in n.phi:
values = n.phi[key]
prePhi = PhiPairLLVM(self.dispatch(Name(values[0])),
entryL)
postPhi = PhiPairLLVM(self.dispatch(Name(values[1])),
loopB[-1].label)
target = VarLLVM(LocalLLVM(key), DEFAULTTYPE)
phis += [phiLLVM(target, [prePhi, postPhi])]
testprepB = self.dispatch(n.testss, None)
testprepB[0].instrs = phis + testprepB[0].instrs
testprepB[0].label = testprepL
testprepB[-1].instrs[-1].defaultDest = testL
# Test Block
loopS = SwitchPairLLVM(DEFAULTTRUE, loopL)
testI = []
testI += [switchLLVM(testVal, elseL, [loopS])]
testB = [blockLLVM(testL, testI)]
# Else Block
if n.else_:
elseB = self.dispatch(n.else_, None)
# Patch in proper label for first block
elseB[0].label = elseL
# Patch in proper destination in last block
elseB[-1].instrs[-1].defaultDest = endL
else:
elseI = []
elseI += [switchLLVM(DEFAULTZERO, endL, [])]
elseB = [blockLLVM(elseL, elseI)]
# Test Block
endI = []
endI += [switchLLVM(DEFAULTZERO, DUMMYL, [])]
endB = [blockLLVM(endL, endI)]
return entryB + testprepB + testB + loopB + elseB + endB
def schedules(org_label, project_label, schedule_label):
''' schedule configuration management
/organizations/aquaya/projects/water-quality/schedules
: viewing a list of this project's schedules
/organizations/aquaya/projects/water-quality/schedules?create=true
: create a new schedule config, immediately redirect to editing
/organizations/aquaya/projects/water-quality/schedules/weekly
: view the 'weekly' schedule config
/organizations/aquaya/projects/water-quality/schedules/weekly?edit=true
: edit a schedule config; accepts GET or POST
'''
user = User.objects(email=session['email'])[0]
orgs = Organization.objects(label=org_label)
if not orgs:
flash('Organization "%s" not found, sorry!' % org_label, 'warning')
return redirect(url_for('organizations'))
org = orgs[0]
# permission-check
if org not in user.organizations and not user.admin_rights:
app.logger.error('%s tried to view a project but was \
denied for want of admin rights' % session['email'])
abort(404)
# find the project
projects = Project.objects(label=project_label, organization=org)
if not projects:
flash('Project "%s" not found, sorry!' % project_label, 'warning')
return redirect(url_for('organizations', org_label=org.label))
project = projects[0]
if request.method == 'POST':
# we have a schedule_label
schedules = Schedule.objects(label=schedule_label, project=project)
if not schedules:
abort(404)
schedule = schedules[0]
form_type = request.form.get('form_type', '')
if form_type == 'info':
if schedule.name != request.form.get('name', ''):
name = request.form.get('name', '')
schedule.name = name
schedules = Schedule.objects(project=project).only('label')
labels = [s.label for s in schedules]
schedule.label = utilities.generate_label(name, labels)
schedule.description = request.form.get('description', '')
elif form_type == 'items':
# modify schedule attachments
# 'items' is of the form report_id__4abcd00123 or statistic_id..
# convert these to real objects
items = request.form.getlist('items')
attached_reports, attached_statistics = [], []
for item in items:
prefix, item_id = item.split('__')
item_type = prefix.split('_')[0]
if item_type == 'report':
reports = Report.objects(id=item_id)
if not reports:
abort(404)
attached_reports.append(reports[0])
elif item_type == 'statistic':
statistics = Statistic.objects(id=item_id)
if not statistics:
abort(404)
attached_statistics.append(statistics[0])
schedule.update(set__reports=attached_reports)
schedule.update(set__statistics=attached_statistics)
# whether we're also sending project data
send_data = request.form.get('send_project_data', '')
if send_data == 'true':
schedule.update(set__send_project_data = True)
else:
schedule.update(set__send_project_data = False)
# save the list of project-data-filters
# filters are formatted like 'filter_id__4abcd123'
filter_ids = request.form.getlist('filters')
attached_filters = []
for filter_id in filter_ids:
filters = Filter.objects(id=filter_id.split('__')[1])
if not filters:
abort(404)
attached_filters.append(filters[0])
schedule.update(set__data_filters = attached_filters)
# how to apply the filters
apply_any_filters = request.form.get('apply_any_filters', '')
if apply_any_filters == 'true':
schedule.update(set__apply_any_filters= True)
else:
schedule.update(set__apply_any_filters= False)
flash('Schedules attachments saved.', 'success')
url = url_for('schedules', org_label=org.label
, project_label=project.label
, schedule_label=schedule.label, edit='true')
return redirect(url + '#items')
elif form_type == 'schedule':
old_interval = schedule.interval
new_interval = {
'type': request.form.get('schedule_type', '')
, 'at': request.form.get('send_at', '')
, 'on_day': request.form.get('send_on_day', '')}
if old_interval != new_interval:
schedule.interval = new_interval
schedule.save()
schedule.reload()
# delete any old jobs and schedule the new ones
update_scheduled_send(schedule.id)
flash('Scheduling interval saved successfully.', 'success')
url = url_for('schedules', org_label=org.label
, project_label=project.label
, schedule_label=schedule.label, edit='true')
return redirect(url + '#schedule')
elif form_type == 'message':
message_type = request.form.get('message_type', '')
schedule.message_type = message_type
if message_type == 'email':
# attach email info to schedule
schedule.email_subject = request.form.get('email_subject', '')
schedule.email_body = request.form.get('email_body', '')
add_email = request.form.get('add_email_recipient_email', '')
if add_email:
# is the email already in place? (shouldn't be)
for recipient in schedule.email_recipients:
if recipient['email'] == add_email:
flash('"%s" is already included in this \
schedule' % add_email, 'warning')
url = url_for('schedules', org_label=org.label
, project_label=project.label
, schedule_label=schedule.label, edit='true')
return redirect(url + '#message')
else:
# add the recipient to the schedule
add_name = request.form.get(
'add_email_recipient_name', '')
recipient = {'email': add_email, 'name': add_name}
schedule.update(push__email_recipients=recipient)
flash('Successfully added "%s" to this schedule' \
% add_email, 'success')
remove_email = request.form.get('remove_email_recipient', '')
if remove_email:
# is the email already attached? (should be)
emails = [r['email'] for r in schedule.email_recipients]
if remove_email not in emails:
flash('"%s" is not included in this schedule and \
cannot be removed' % remove_email, 'warning')
url = url_for('schedules', org_label=org.label
, project_label=project.label
, schedule_label=schedule.label, edit='true')
return redirect(url + '#message')
else:
# remove it
for recipient in schedule.email_recipients:
if recipient['email'] == remove_email:
schedule.update(
pull__email_recipients=recipient)
flash('Successfully removed the address "%s" \
from this schedule' % remove_email
, 'success')
elif message_type == 'sms':
# attach sms info to schedule
add_number = request.form.get(
'add_sms_recipient_phone_number', '')
if add_number:
# is the number already in place? (shouldn't be)
for recipient in schedule.sms_recipients:
if recipient['phone_number'] == add_number:
flash('"%s" is already included in this \
schedule' % add_number, 'warning')
url = url_for('schedules', org_label=org.label
, project_label=project.label
, schedule_label=schedule.label, edit='true')
return redirect(url + '#message')
else:
# add the recipient to the schedule
add_name = request.form.get(
'add_sms_recipient_name', '')
recipient = {'phone_number': add_number
, 'name': add_name}
schedule.update(push__sms_recipients=recipient)
flash('Successfully added "%s" to this schedule' \
% add_number, 'success')
remove_number = request.form.get('remove_sms_recipient', '')
if remove_number:
# is the number already attached? (should be)
numbers = [r['phone_number'] for r in \
schedule.sms_recipients]
if remove_number not in numbers:
flash('"%s" is not included in this SMS schedule and \
cannot be removed' % remove_number, 'warning')
url = url_for('schedules', org_label=org.label
, project_label=project.label
, schedule_label=schedule.label, edit='true')
return redirect(url + '#message')
else:
# remove it
for recipient in schedule.sms_recipients:
if recipient['phone_number'] == remove_number:
schedule.update(
pull__sms_recipients=recipient)
flash('Successfully removed the address "%s" \
from this schedule' % remove_number
, 'success')
break
schedule.save()
flash('Message settings changed successfully.', 'success')
url = url_for('schedules', org_label=org.label
, project_label=project.label, schedule_label=schedule.label
, edit='true')
return redirect(url + '#message')
elif form_type == 'admin':
# delete the schedule
# keeping the associated messages as they're an important log
# related items in the queue will not fire sans schedule
name = schedule.name
# cancel any outstanding jobs
if schedule.next_task_id:
redis_config = app.config['REDIS_CONFIG']
use_connection(Redis(redis_config['host'], redis_config['port']
, password=redis_config['password']))
scheduler = Scheduler()
job = Job(id=schedule.next_task_id)
scheduler.cancel(job)
# delete the job itself
schedule.delete()
app.logger.info('%s deleted schdedule "%s"' % \
(session['email'], name))
flash('schedule "%s" was deleted successfully' % name, 'success')
return redirect(url_for('schedules', org_label=org.label
, project_label=project.label))
else:
# bad 'form_type'
abort(404)
try:
schedule.save()
flash('Changes saved successfully.', 'success')
return redirect(url_for('schedules', org_label=org.label
, project_label=project.label
, schedule_label=schedule.label, edit='true'))
except:
app.logger.error('%s experienced an error saving info about %s' % (
session['email'], request.form['name']))
flash('Error saving changes - make sure schedule names are \
unique.', 'error')
return redirect(url_for('schedules', org_label=org.label
, project_label=project.label, schedule_label=schedule_label
, edit='true'))
if request.method == 'GET':
if schedule_label:
schedules = Schedule.objects(label=schedule_label, project=project)
if not schedules:
app.logger.error('%s tried to access a schedule that does \
not exist' % session['email'])
flash('schedule "%s" not found, sorry!' % schedule_label
, 'warning')
return redirect(url_for('schedules', org_label=org.label
, project_label=project.label))
schedule = schedules[0]
if request.args.get('edit', '') == 'true':
available_reports = Report.objects(project=project)
available_filters = Filter.objects(project=project)
available_statistics = Statistic.objects(project=project)
emails = json.dumps(schedule.email_recipients)
current_time = datetime.datetime.utcnow()
return render_template('schedule_edit.html', schedule=schedule
, available_reports=available_reports
, available_filters=available_filters
, available_statistics=available_statistics
, email_recipients=emails
, current_time = current_time)
elif request.args.get('fire', '') == 'true':
if ready_to_send(schedule):
# immediately enqueue the job with rq
# countdown briefly to ensure next_task_id can be updated?
redis_config = app.config['REDIS_CONFIG']
use_connection(Redis(redis_config['host']
, redis_config['port']
, password=redis_config['password']))
queue = Queue()
job = queue.enqueue(send_scheduled_report, schedule.id)
# save the id
# race condition with enqueued func above?
# that's why we used to have a countdown
schedule.update(set__next_task_id = job.id)
flash('Sending message, this may take a few \
moments. Check the message log for updates.', 'info')
else:
flash('This schedule is not properly configured.', 'error')
return redirect(url_for('schedules', org_label=org.label
, project_label=project.label
, schedule_label=schedule_label))
else:
# get all messages attached to this schedule
messages = Message.objects(schedule=schedule).order_by(
'-creation_time')
return render_template('schedule.html', schedule=schedule
, messages=messages)
if request.args.get('create', '') == 'true':
# create a new schedule
# CSRF validation
token = request.args.get('token', '')
if not verify_token(token):
abort(403)
try:
schedule_name = 'schd-%s' \
% utilities.generate_random_string(6)
new_schedule = Schedule(
label = schedule_name.lower()
, project = project
, name = schedule_name
)
new_schedule.save()
app.logger.info('schedule created by %s' % session['email'])
flash('schedule created. Please change the default values.'
, 'success')
except:
app.logger.error('schedule creation failed for %s' % \
session['email'])
flash('There was an error, sorry :/', 'error')
return redirect(url_for('projects', org_label=org.label
, project=project.label))
# redirect to the editing screen
return redirect(url_for('schedules', org_label=org.label
, project_label=project.label
, schedule_label=new_schedule.label, edit='true'))
# no statistic in particular was specified; show em all
schedules = Schedule.objects(project=project)
return render_template('project_schedules.html', project=project
, schedules=schedules)
def reports(org_label, project_label, report_label):
''' creating reports
/organizations/aquaya/projects/water-quality/reports
: viewing a list of this project's reports
/organizations/aquaya/projects/water-quality/reports?create=true
: create a new report config, immediately redirect to editing
/organizations/aquaya/projects/water-quality/reports/submitters
: view a report
/organizations/aquaya/projects/water-quality/reports/submitters?edit=true
: edit a report; accepts GET or POST
'''
user = User.objects(email=session['email'])[0]
orgs = Organization.objects(label=org_label)
if not orgs:
flash('Organization "%s" not found, sorry!' % org_label, 'warning')
return redirect(url_for('organizations'))
org = orgs[0]
# permission-check
if org not in user.organizations and not user.admin_rights:
app.logger.error('%s tried to view a project but was \
denied for want of admin rights' % session['email'])
abort(404)
# find the project
projects = Project.objects(label=project_label, organization=org)
if not projects:
flash('Project "%s" not found, sorry!' % project_label, 'warning')
return redirect(url_for('organizations', org_label=org.label))
project = projects[0]
if request.method == 'POST':
# we have a report_label
reports = Report.objects(label=report_label, project=project)
if not reports:
abort(404)
report = reports[0]
form_type = request.form.get('form_type', '')
if form_type == 'info':
if report.name != request.form.get('name', ''):
name = request.form.get('name', '')
report.update(set__name = name)
reports = Report.objects(project=project).only('label')
labels = [r.label for r in reports]
report.update(set__label = utilities.generate_label(name
, labels))
report.update(set__description = request.form.get('description'
, ''))
report.reload()
flash('Report details saved successfully.', 'success')
return redirect(url_for('reports', org_label=org.label
, project_label=project.label, report_label=report.label
, edit='true'))
elif form_type == 'components':
# attach components to report
# 'items' is of the form graph_id__4abcd00123 or statistic_id..
# convert these to full objects
items = request.form.getlist('items')
components = []
for item in items:
prefix, item_id = item.split('__')
item_type = prefix.split('_')[0]
if item_type == 'graph':
graphs = Graph.objects(id=item_id)
if not graphs:
abort(404)
components.append(graphs[0])
elif item_type == 'statistic':
statistics = Statistic.objects(id=item_id)
if not statistics:
abort(404)
components.append(statistics[0])
report.update(set__components=components)
flash('Report components saved successfully.', 'success')
url = url_for('reports', org_label=org.label
, project_label=project.label, report_label=report.label
, edit='true')
return redirect(url + '#components')
elif form_type == 'admin':
# delete the report
name = report.name
report.delete()
app.logger.info('%s deleted report "%s"' % \
(session['email'], name))
flash('report "%s" was deleted successfully' % name, 'success')
return redirect(url_for('reports', org_label=org.label
, project_label=project.label))
else:
# bad 'form_type'
abort(404)
if request.method == 'GET':
if report_label:
reports = Report.objects(label=report_label, project=project)
if not reports:
app.logger.error('%s tried to access a report that does \
not exist' % session['email'])
flash('Report "%s" not found, sorry!' % report_label
, 'warning')
return redirect(url_for('projects', org_label=org.label
, project_label=project.label))
report = reports[0]
if request.args.get('edit', '') == 'true':
# extract the component names for js typeahead
available_graphs = Graph.objects(project=project)
available_statistics = Statistic.objects(project=project)
return render_template('report_edit.html'
, report=report
, available_graphs = available_graphs
, available_statistics = available_statistics)
elif request.args.get('preview', '') == 'true':
# create pdf preview of the report
# calculate all statistics and graph datasets
items = _generate_report_items(report)
# get the current time
current_time = datetime.datetime.utcnow()
return render_template('report_preview.html', report=report
, items=items, current_time=current_time)
elif request.args.get('render', '') == 'true':
''' create PDF versions of the preview
also send them to s3
'''
# CSRF validation
token = request.args.get('token', '')
if not verify_token(token):
abort(403)
utilities.create_rendering(report)
flash('The report is now being rendered, this process may \
take a few moments. Check the table below for \
updates', 'info')
return redirect(url_for('reports', org_label=org.label
, project_label=project.label, report_label=report.label))
else:
# find all relevant renderings
renderings = Rendering.objects(report=report).order_by(
'-creation_time')
return render_template('report.html', report=report
, renderings=renderings)
if request.args.get('create', '') == 'true':
# create a new report
# CSRF validation
token = request.args.get('token', '')
if not verify_token(token):
abort(403)
try:
report_name = 'report-%s' \
% utilities.generate_random_string(6)
new_report= Report(
creation_time = datetime.datetime.utcnow()
, creator = user
, label = report_name.lower()
, project = project
, name = report_name
, public_label = utilities.generate_random_string(24)
)
new_report.save()
app.logger.info('report created by %s' % session['email'])
flash('Report created, please change the default values.'
, 'success')
except:
app.logger.error('report creation failed for %s' % \
session['email'])
flash('There was an error, sorry :/', 'error')
return redirect(url_for('projects', org_label=org.label
, project=project.label))
# redirect to the editing screen
return redirect(url_for('reports', org_label=org.label
, project_label=project.label
, report_label=new_report.label, edit='true'))
# no statistic in particular was specified; show em all
reports = Report.objects(project=project)
return render_template('project_reports.html', project=project
, reports=reports)
def graphs(org_label, project_label, graph_label):
''' graphin things
/organizations/aquaya/projects/water-quality/graphs
: view a list of all graphs for the project
/organizations/aquaya/projects/water-quality/graphs?create=true
: create a new graph config, immediately redirect to editing
/organizations/aquaya/projects/water-quality/graphs/ph-vs-time
: view a graph
/organizations/aquaya/projects/water-quality/graphs/ph-vs-time?edit=true
: edit a graph; accepts GET or POST
'''
user = User.objects(email=session['email'])[0]
orgs = Organization.objects(label=org_label)
if not orgs:
flash('Organization "%s" not found, sorry!' % org_label, 'warning')
return redirect(url_for('organizations'))
org = orgs[0]
# permission-check
if org not in user.organizations and not user.admin_rights:
app.logger.error('%s tried to view a project but was \
denied for want of admin rights' % session['email'])
abort(404)
# find the project
projects = Project.objects(label=project_label, organization=org)
if not projects:
flash('Project "%s" not found, sorry!' % project_label, 'warning')
return redirect(url_for('organizations', org_label=org.label))
project = projects[0]
if request.method == 'POST':
# we have a graph_label
graphs = Graph.objects(label=graph_label, project=project)
if not graphs:
abort(404)
graph = graphs[0]
form_type = request.form.get('form_type', '')
if form_type == 'info':
if graph.name != request.form.get('name', ''):
name = request.form.get('name', '')
graph.update(set__name = name)
graphs = Graph.objects(project=project).only('label')
labels = [g.label for g in graphs]
graph.update(set__label = utilities.generate_label(name
, labels))
# reload to catch the name change
graph.reload()
graph.update(set__description =
request.form.get('description', ''))
graph.update(set__graph_type = request.form.get('graph_type', ''))
# axes specify a header and come of the form 'header_id__4abcd001'
xaxis = request.form.get('xaxis', '')
if xaxis:
xaxis = xaxis.split('header_id__')[1]
header = Header.objects(id=xaxis)[0]
graph.update(set__xaxis = header)
yaxis = request.form.get('yaxis', '')
if yaxis:
yaxis = yaxis.split('header_id__')[1]
header = Header.objects(id=yaxis)[0]
graph.update(set__yaxis = header)
# pie chart headers are similar to axes..
pie_header = request.form.get('pie_header', '')
if pie_header:
pie_header = pie_header.split('header_id__')[1]
header = Header.objects(id=pie_header)[0]
graph.update(set__pie_header = header)
elif form_type == 'filters':
# extract the 'any filters' vs 'all' distinction
filter_settings = request.form.get('apply_any_filters', '')
if filter_settings == 'true':
graph.update(set__apply_any_filters = True)
else:
graph.update(set__apply_any_filters = False)
# attach filter to graph
requested_filter_ids = request.form.getlist('filters')
attached_filters = []
for requested_id in requested_filter_ids:
prefix, filter_id = requested_id.split('__')
filters = Filter.objects(id=filter_id)
if not filters:
abort(404)
attached_filters.append(filters[0])
graph.update(set__filters = attached_filters)
elif form_type == 'admin':
# delete the graph
name = graph.name
utilities.delete_graph(graph, session['email'])
flash('graph "%s" was deleted successfully' % name, 'success')
return redirect(url_for('graphs', org_label=org.label
, project_label=project.label))
else:
# bad 'form_type'
abort(404)
flash('changes saved successfully', 'success')
return redirect(url_for('graphs', org_label=org.label
, project_label=project.label, graph_label=graph.label))
if request.method == 'GET':
if graph_label:
graphs = Graph.objects(label=graph_label, project=project)
if not graphs:
app.logger.error('%s tried to access a graph that does not \
exist' % session['email'])
flash('Graph "%s" not found, sorry!' % graph_label
, 'warning')
return redirect(url_for('projects'), org_label=org.label
, project_label=project.label)
graph = graphs[0]
if request.args.get('edit', '') == 'true':
# valid graph types
graph_types = ['line', 'scatter', 'bar', 'chart', 'pie']
available_filters = Filter.objects(project=project)
return render_template('graph_edit.html', graph=graph
, graph_types = graph_types
, allowed_graph_types = constants.graph_types
, available_filters = available_filters)
else:
# render a graph
data = []
project_count = None
filtered_count = None
if graph.graph_type == 'line':
if graph.xaxis and graph.yaxis:
data, project_count = (
utilities.generate_line_graph_data(graph))
filtered_count = len(data)
else:
flash('define an x-axis and y-axis for plotting'
, 'warning')
elif graph.graph_type == 'pie':
if graph.pie_header:
data, project_count = (
utilities.generate_pie_chart_data(graph))
filtered_count = sum([i['data'] for i in json.loads(data)])
else:
flash('define a column to create this pie chart'
, 'warning')
return render_template('graph.html', graph=graph, data=data
, project_count=project_count
, filtered_count = filtered_count)
if request.args.get('create', '') == 'true':
# create a new graph
# CSRF validation
token = request.args.get('token', '')
if not verify_token(token):
abort(403)
try:
graph_name = 'graph-%s' % utilities.generate_random_string(6)
new_graph = Graph(
creation_time = datetime.datetime.utcnow()
, creator = user
, label = graph_name.lower()
, project = project
, name = graph_name
)
new_graph.save()
app.logger.info('graph created by %s' % session['email'])
flash('graph created; please change the defaults', 'success')
except:
app.logger.error('graph creation failed for %s' % \
session['email'])
flash('There was an error, sorry :/', 'error')
return redirect(url_for('projects', org_label=org.label
, project=project.label))
# redirect to the editing screen
return redirect(url_for('graphs', org_label=org.label
, project_label=project.label, graph_label=new_graph.label
, edit='true'))
# no graph in particular was specified
graphs = Graph.objects(project=project)
return render_template('project_graphs.html', project=project
, graphs=graphs)
def uploads(o_label, p_label, u_label):
''' uploadin
/organizations/aquaya/projects/water-quality/uploads?create=true
: create a new upload, immediately redirect to editing
/organizations/aquaya/projects/water-quality/uploads/may-2012
: view an upload
/organizations/aquaya/projects/water-quality/uploads/may-2012?edit=true
: edit an upload; accepts GET or POST
'''
# can't wrap the line through a route or spaces are injected
# hence this silliness:
org_label = o_label
project_label = p_label
upload_label = u_label
user = User.objects(email=session['email'])[0]
orgs = Organization.objects(label=org_label)
if not orgs:
flash('Organization "%s" not found, sorry!' % org_label, 'warning')
return redirect(url_for('organizations'))
org = orgs[0]
# permission-check
if org not in user.organizations and not user.admin_rights:
app.logger.error('%s tried to view a project but was \
denied for want of admin rights' % session['email'])
abort(404)
projects = Project.objects(label=project_label, organization=org)
if not projects:
flash('Project "%s" not found, sorry!' % project_label, 'warning')
return redirect(url_for('organizations', org_label=org.label))
project = projects[0]
if request.method == 'GET':
if not upload_label and request.args.get('create', '') == 'true':
# create a new upload
# CSRF validation
token = request.args.get('token', '')
if not verify_token(token):
abort(403)
try:
name = 'upl-%s' % utilities.generate_random_string(6)
new_upload = Upload(
label = name.lower()
, name = name
, project = project
)
new_upload.save()
project.update(push__uploads=new_upload)
app.logger.info('upload created by %s' % session['email'])
flash('please select a file and make other edits', 'warning')
except:
app.logger.error('upload creation failed for %s' % \
session['email'])
flash('There was an error, sorry :/', 'error')
return redirect(url_for('projects', org_label=org.label
, project_label=project.label))
# redirect to the editing screen
return redirect(url_for('uploads', o_label=org.label
, p_label=project.label, u_label=new_upload.label
, edit='true'))
elif not upload_label:
# could show projects uploads but punting for now
unique_entries = Entry.objects(project=project, unique=True
, visible=True)
duplicate_entries = Entry.objects(project=project
, unique=False)
hidden_entries = Entry.objects(project=project, visible=False
, unique=True)
upload_entry_counts = {}
for upload in project.uploads:
upload_entry_counts[upload.id] = Entry.objects(upload=upload).count()
return render_template('project_uploads.html', project=project
, unique_entries=unique_entries
, duplicate_entries=duplicate_entries
, upload_entry_counts=upload_entry_counts
, hidden_entries=hidden_entries)
else:
# we have an upload label
uploads = Upload.objects(label=upload_label, project=project)
if not uploads:
flash('Upload "%s" not found, sorry!' % upload_label
, 'warning')
return redirect(url_for('projects', org_label=org.label
, project_label=project.label))
upload = uploads[0]
if request.args.get('edit', '') == 'true':
# show the editor
# queue up any flashed messages
# these get set during the worker's processing of a file
worker_messages = upload.worker_messages
for message in worker_messages:
flash(message['message'], message['status'])
upload.update(set__worker_messages = [])
return render_template('upload_edit.html', upload=upload
, number_of_pages=0)
else:
if not upload.filename:
# need to add one so redirect to editing
flash('please specify a file for this upload', 'warning')
return redirect(url_for('uploads', o_label=org.label
, p_label=project.label, u_label=upload.label
, edit='true'))
else:
# things are still in progress
if not upload.s3_key:
flash('Your file is being processed in the '
'background. Refresh the page to see updates.'
, 'info')
return render_template('upload.html', upload=upload
, number_of_pages=0)
# otherwise upload is complete, show the data
# pagination
total_entries = Entry.objects(project=project
, upload=upload).count()
entries_per_page = 10
pages = utilities.calculate_pages(total_entries
, entries_per_page=entries_per_page)
# validate the requested page
current_page = utilities.validate_page_request(
request.args.get('page', 1), pages)
# get the sorted entries
entries = utilities.query_entries(project
, upload = upload
, skip = (entries_per_page * (current_page - 1))
, limit = entries_per_page)
hidden_entries_count = Entry.objects(upload=upload
, visible=False).count()
# queue up any flashed messages
# these get set during the worker's processing of a file
worker_messages = upload.worker_messages
for message in worker_messages:
flash(message['message'], message['status'])
upload.update(set__worker_messages = [])
return render_template('upload.html'
, upload=upload
, entries=entries
, total_entries = total_entries
, hidden_entries_count=hidden_entries_count
, current_page = current_page
, number_of_pages = pages)
elif request.method == 'POST':
# we have an upload label
uploads = Upload.objects(label=upload_label, project=project)
if not uploads:
abort(404)
upload = uploads[0]
form_type = request.form.get('form_type', '')
if form_type == 'info':
if upload.name != request.form.get('name', ''):
name = request.form.get('name', '')
upload.update(set__name = name)
uploads = Upload.objects(project=project).only('label')
labels = [u.label for u in uploads]
upload.update(set__label =
utilities.generate_label(name, labels))
upload.project.update(set__update_time =
datetime.datetime.utcnow())
upload.reload()
if upload.description != request.form.get('description', ''):
upload.update(set__description =
request.form.get('description', ''))
upload.project.update(set__update_time =
datetime.datetime.utcnow())
if request.files.get('data_file', ''):
data_file = request.files.get('data_file')
try:
filename = uploaded_data.save(data_file)
if '..' in filename or filename.startswith('/'):
app.logger.info('%s tried to upload a malicious file \
"%s"' % (session['email'], filename))
flash('bad filename, sorry :/', 'error')
return redirect(url_for('uploads'
, o_label=org.label, p_label=project.label
, u_label=upload.label, edit='true'))
absolute_filename = uploaded_data.path(filename)
except UploadNotAllowed:
app.logger.info('%s tried to upload an unsupported file'
% session['email'])
flash('we currently only support .xls files, sorry :/'
, 'error')
return redirect(url_for('uploads'
, o_label=org.label, p_label=project.label
, u_label=upload.label, edit='true'))
upload.update(set__filename =
os.path.basename(absolute_filename))
upload.update(set__uploaded_by = user)
# enqueue upload-processing
redis_config = app.config['REDIS_CONFIG']
use_connection(Redis(redis_config['host'], redis_config['port']
, password=redis_config['password']))
queue = Queue()
queue.enqueue(process_uploaded_file, upload.id,
absolute_filename)
elif form_type == 'admin':
# delete the upload
name = upload.name
# pull out the upload from the parent project first
project.update(pull__uploads=upload)
# delete associated entries, remove from system, remove from s3
deleted_entries_count = utilities.delete_upload(upload
, session['email'])
flash('upload "%s" was deleted successfully; %s entries deleted \
as well' % (name, deleted_entries_count), 'success')
# see if, given this deletion, the dupes are still dupes
unduped_count = utilities.recheck_duplicate_entries(project)
if unduped_count:
flash('%s entries were automatically added to the project \
as they are no longer duplicates' % unduped_count, 'info')
return redirect(url_for('uploads', o_label=org.label
, p_label=project.label))
else:
# bad 'form_type'
abort(404)
try:
upload.save()
project.save()
except:
app.logger.error('%s experienced an error saving info about the \
upload %s' % (session['email'], request.form['name']))
flash('error, make sure names are unique', 'error')
return redirect(url_for('projects'
, org_label=upload.project.organization.label
, project_label=upload.project.label))
return redirect(url_for('uploads'
, o_label=upload.project.organization.label, p_label=project.label
, u_label=upload.label))
def organizations(org_label):
''' show the organizations
if there's a label included in the route, render that organization alone
'''
user = User.objects(email=session['email'])[0]
if request.method == 'POST':
orgs = Organization.objects(label=org_label)
if not orgs:
abort(404)
org = orgs[0]
# permission-check
if org not in user.organizations and not user.admin_rights:
app.logger.error('%s tried to edit an organization but was \
denied for want of admin rights' % session['email'])
abort(404)
form_type = request.form.get('form_type', '')
if form_type == 'info':
if org.name != request.form.get('name', ''):
name = request.form.get('name', '')
org.name = name
orgs = Organization.objects().only('label')
labels = [o.label for o in orgs]
org.label = utilities.generate_label(name, labels)
if org.description != request.form.get('description', ''):
app.logger.info('%s edited the description of %s to %s' % (
session['email'], org.name
, request.form.get('description', '')))
org.description = request.form.get('description', '')
if org.location != request.form.get('location', ''):
app.logger.info('%s edited the location of %s to %s' % (
session['email'], org.name
, request.form.get('location', '')))
org.location = request.form.get('location', '')
elif form_type == 'add_members':
# push membership
target = request.form.get('add_member_email', '')
new_members = User.objects(email=target)
if not new_members:
flash('we cannot find "%s", has it been registered?' % \
target, 'error')
return redirect(url_for('organizations', org_label=org.label))
new_member = new_members[0]
# already a member?
if org in new_member.organizations:
flash('"%s" is already a member of "%s"' % (target, org.name)
, 'warning')
return redirect(url_for('organizations', org_label=org.label))
else:
# add them
new_member.update(push__organizations=org)
flash('successfully added "%s" to "%s"' % (target, org.name)
, 'success')
return redirect(url_for('organizations', org_label=org.label))
elif form_type == 'remove_members':
# push/pull membership
target = request.form.get('remove_member_email', '')
old_members = User.objects(email=target)
if not old_members:
flash('we cannot find "%s", has it been registered?' % \
target, 'error')
return redirect(url_for('organizations', org_label=org.label))
old_member = old_members[0]
# not yet a member?
if org not in old_member.organizations:
flash('"%s" is not yet a member of "%s"' % (target, org.name)
, 'warning')
return redirect(url_for('organizations', org_label=org.label))
else:
# drop 'em
old_member.update(pull__organizations=org)
flash('successfully removed "%s" from %s' % (target, org.name)
, 'info')
return redirect(url_for('organizations', org_label=org.label))
elif form_type == 'admin':
# delete the organization; permission-check first
if not user.admin_rights:
app.logger.error('%s tried to delete %s but was denied for \
want of admin rights' % (session['email'], org.name))
abort(404)
# revoke membership first
members = User.objects(organizations=org)
for member in members:
member.update(pull__organizations=org)
# blow away the org itself
name = org.name
org.delete()
app.logger.info('%s deleted %s' % (session['email'], name))
flash('organization "%s" was deleted' % name, 'success')
return redirect(url_for('organizations'))
else:
# bad 'form_type'
abort(404)
try:
org.save()
flash('changes saved successfully', 'success')
except:
org.reload()
app.logger.error('%s experienced an error saving info about %s' % (
session['email'], request.form['name']))
flash('Error saving changes, is the name unique?', 'error')
return redirect(url_for('organizations', org_label=org.label
, edit='true'))
if request.method == 'GET':
if org_label:
orgs = Organization.objects(label=org_label)
if not orgs:
app.logger.error('%s tried to access an organization that \
does not exist' % session['email'])
flash('Organization "%s" not found, sorry!' % org_label
, 'warning')
return redirect(url_for('organizations'))
org = orgs[0]
# permission-check
if org not in user.organizations and not user.admin_rights:
app.logger.error('%s tried to access an organization but was \
denied for want of admin rights' % session['email'])
abort(404)
if request.args.get('edit', '') == 'true':
users = User.objects(organizations=org)
return render_template('organization_edit.html'
, organization=org, users=users)
else:
# get all the members
users = User.objects(organizations=org)
return render_template('organization.html', organization=org
, users=users)
if request.args.get('create', '') == 'true':
# create a new form
# permissions-check
if not user.admin_rights:
app.logger.error('%s tried to create an organization but was \
denied for want of admin rights' % session['email'])
abort(404)
# CSRF validation
token = request.args.get('token', '')
if not verify_token(token):
abort(403)
try:
org_name = 'org-%s' % utilities.generate_random_string(6)
new_org = Organization(
label = org_name.lower()
, name = org_name
)
new_org.save()
app.logger.info('organization created by %s' % \
session['email'])
flash('organization created; please change the defaults',
'success')
except:
app.logger.error('organization creation failed for %s' % \
session['email'])
flash('There was an error in the form, sorry :/', 'error')
return redirect(url_for('organizations'))
# redirect to the editing screen
return redirect(url_for('organizations', org_label=new_org.label
, edit='true'))
# nobody in particular was specified; show em all
if user.admin_rights:
orgs = Organization.objects()
else:
orgs = user.organizations
# find the users for each org
users = {}
for org in orgs:
users[org.name] = User.objects(organizations=org)
return render_template('organizations.html', organizations=orgs
, users=users)
def filters(org_label, project_label, filter_label):
''' creating filters for various purposes
/organizations/aquaya/projects/water-quality/filters?create=true
: create a new filter config, immediately redirect to editing
/organizations/aquaya/projects/water-quality/filters/big-cities
: view a filter
/organizations/aquaya/projects/water-quality/filters/big-cities?edit=true
: edit a filter; accepts GET or POST
'''
user = User.objects(email=session['email'])[0]
orgs = Organization.objects(label=org_label)
if not orgs:
flash('Organization "%s" not found, sorry!' % org_label, 'warning')
return redirect(url_for('organizations'))
org = orgs[0]
# permission-check
if org not in user.organizations and not user.admin_rights:
app.logger.error('%s tried to view a project but was \
denied for want of admin rights' % session['email'])
abort(404)
# find the project
projects = Project.objects(label=project_label, organization=org)
if not projects:
flash('Project "%s" not found, sorry!' % project_label, 'warning')
return redirect(url_for('organizations', org_label=org.label))
project = projects[0]
if request.method == 'POST':
# we have a filter_label
filters= Filter.objects(label=filter_label, project=project)
if not filters:
abort(404)
# should try to avoid overriding the builtin..
filter = filters[0]
form_type = request.form.get('form_type', '')
if form_type == 'info':
if filter.name != request.form.get('name', ''):
name = request.form.get('name', '')
filter.name = name
filters = Filter.objects(project=project).only('label')
labels = [f.label for f in filters]
filter.label = utilities.generate_label(name, labels)
filter.description = request.form.get('description', '')
filter.comparison = request.form.get('comparison', '')
# header of the format header_id__4abcd0001
header_id = request.form.get('header', '')
header_id = header_id.split('header_id__')[1]
header = Header.objects(id=header_id)[0]
filter.header = header
if header.data_type == 'datetime':
# check if relative or absolute datetime comparison
if filter.comparison in \
constants.filter_comparisons['datetime_absolute']:
compare_to = '%s %s' % (
request.form.get('date_compare_to', '')
, request.form.get('time_compare_to', ''))
try:
dt = datetime.datetime.strptime(
compare_to, '%B %d, %Y %I:%M %p')
filter.compare_to = {'value': dt}
except:
flash("didn't understand that date formatting; make \
sure it looks like 'June 21, 2012' and \
'02:45 PM'", 'error')
return redirect(url_for('filters', org_label=org.label
, project_label=project.label
, filter_label=filter.label
, edit='true'))
elif filter.comparison in \
constants.filter_comparisons['datetime_relative']:
inputs = [request.form.get('relative_years', '')
, request.form.get('relative_months', '')
, request.form.get('relative_weeks', '')
, request.form.get('relative_days', '')
, request.form.get('relative_hours', '')
, request.form.get('relative_minutes', '')]
parsed_inputs = []
for i in inputs:
if i:
parsed_inputs.append(i)
else:
parsed_inputs.append(0)
# verify that we have numbers
try:
values = [int(i) for i in parsed_inputs]
except:
flash('input times must be whole numbers', 'error')
return redirect(url_for('filters', org_label=org.label
, project_label=project.label
, filter_label=filter.label
, edit='true'))
filter.compare_to = {'value': values}
else:
app.logger.info('unknown comparison "%s" for project %s' \
% (comparison, project.name))
abort(404)
else:
# not datetime; cast the value we're comparing against
compare_to = request.form.get('string_compare_to', '')
filter.compare_to = {'value': utilities.smart_cast(compare_to)}
elif form_type == 'admin':
# delete the filter
name = filter.name
# pull filters out of the statistics
statistics = Statistic.objects(filters=filter)
for statistic in statistics:
statistic.update(pull__filters=filter)
# pull filters out of the graphs
graphs = Graph.objects(filters=filter)
for graph in graphs:
graph.update(pull__filters=filter)
filter.delete()
app.logger.info('%s deleted filter "%s"' % \
(session['email'], name))
flash('filter "%s" was deleted successfully' % name, 'success')
return redirect(url_for('filters', org_label=org.label
, project_label=project.label))
else:
# bad 'form_type'
abort(404)
try:
filter.save()
flash('changes saved successfully', 'success')
return redirect(url_for('filters', org_label=org.label
, project_label=project.label, filter_label=filter.label))
except:
app.logger.error('%s experienced an error saving info about %s' % (
session['email'], request.form['name']))
flash('Error saving changes -- make sure filter names are unique.'
, 'error')
return redirect(url_for('filters', org_label=org.label
, project_label=project.label, filter_label=filter_label
, edit='true'))
if request.method == 'GET':
if filter_label:
filters = Filter.objects(label=filter_label, project=project)
if not filters:
app.logger.error('%s tried to access a filter that does not \
exist' % session['email'])
flash('Filter "%s" not found, sorry!' % filter_label
, 'warning')
return redirect(url_for('projects', org_label=org.label
, project_label=project.label))
filter = filters[0]
if request.args.get('edit', '') == 'true':
# valid comparisons
comparisons = json.dumps(constants.filter_comparisons)
# list of allowed absolute datetime comparisons
absolute_comparisons = \
constants.filter_comparisons['datetime_absolute']
relative_values = None
if filter.comparison in \
constants.filter_comparisons['datetime_relative']:
relative_values = filter.compare_to['value']
return render_template('filter_edit.html', filter=filter
, comparisons=comparisons
, absolute_datetime_comparisons=absolute_comparisons
, relative_datetime_compare_values=relative_values)
else:
# apply the filter; start with some defaults
conditions = {
'project': project
, 'unique': True
, 'visible': True
}
entries = Entry.objects(**conditions)
project_count = len(entries)
if not filter.comparison:
return render_template('filter.html', filter=filter
, project_count=project_count)
# find all matches for this filter to display
matches = utilities.apply_filter(filter, entries)
# sort the matches
matches = utilities.sort_entries(project, matches)
# pagination
entries_per_page = 10
pages = utilities.calculate_pages(len(matches)
, entries_per_page=entries_per_page)
# validate the requested page
current_page = utilities.validate_page_request(
request.args.get('page', 1), pages)
# manually paginate
start_index = entries_per_page * (current_page - 1)
end_index = start_index + entries_per_page
paginated_matches = matches[start_index:end_index]
# list of allowed absolute datetime comparisons
absolute_comparisons = \
constants.filter_comparisons['datetime_absolute']
# if the filter is for a relative datetime, parse the value
relative_value = None
if filter.comparison in \
constants.filter_comparisons['datetime_relative']:
# filter.compare_to is a list [year, month, etc..]
# first non zero value will be used
for i in range(len(filter.compare_to['value'])):
if filter.compare_to['value'][i] != 0:
break
# values in the compare_to list
periods = ['year', 'month', 'week', 'day', 'hour'
, 'minute']
# see if we need to make it plural
suffix = ''
if filter.compare_to['value'][i] > 1:
suffix = 's'
relative_value = '%s %s%s' \
% (filter.compare_to['value'][i], periods[i], suffix)
return render_template('filter.html', filter=filter
, entries = paginated_matches
, total_matches = len(matches)
, project_count=project_count
, current_page = current_page
, number_of_pages = pages
, absolute_datetime_comparisons=absolute_comparisons
, relative_datetime_value=relative_value)
if request.args.get('create', '') == 'true':
# create a new filter
# CSRF validation
token = request.args.get('token', '')
if not verify_token(token):
abort(403)
try:
filter_name = 'filter-%s' % utilities.generate_random_string(6)
new_filter= Filter(
creation_time = datetime.datetime.utcnow()
, creator = user
, label = filter_name.lower()
, project = project
, name = filter_name
)
new_filter.save()
app.logger.info('filter created by %s' % session['email'])
flash('filter created; please change the defaults', 'success')
except:
app.logger.error('filter creation failed for %s' % \
session['email'])
flash('There was an error, sorry :/', 'error')
return redirect(url_for('projects', org_label=org.label
, project=project.label))
# redirect to the editing screen
return redirect(url_for('filters', org_label=org.label
, project_label=project.label, filter_label=new_filter.label
, edit='true'))
# no filter in particular was specified, show 'em all
filters = Filter.objects(project=project)
# list of allowed absolute datetime comparisons
absolute_comparisons = \
constants.filter_comparisons['datetime_absolute']
relative_values = {}
for filter in filters:
# if the filter is for a relative datetime, parse the value
relative_value = None
if filter.comparison in \
constants.filter_comparisons['datetime_relative']:
# filter.compare_to is a list [year, month, etc..]
# first non zero value will be used
for i in range(len(filter.compare_to['value'])):
if filter.compare_to['value'][i] != 0:
break
# values in the compare_to list
periods = ['year', 'month', 'week', 'day', 'hour'
, 'minute']
# see if we need to make it plural
suffix = ''
if filter.compare_to['value'][i] > 1:
suffix = 's'
relative_value = '%s %s%s' % (
filter.compare_to['value'][i], periods[i], suffix)
relative_values[filter.name] = relative_value
return render_template('project_filters.html', project=project
, filters=filters
, absolute_datetime_comparisons=absolute_comparisons
, relative_datetime_values=relative_values)
def statistics(org_label, project_label, statistic_label):
''' creating statistics for various purposes
/organizations/aquaya/projects/water-quality/statistics
: viewing a list of this project's statistics
/organizations/aquaya/projects/water-quality/statistics?create=true
: create a new statistic config, immediately redirect to editing
/organizations/aquaya/projects/water-quality/statistics/mean-pH
: view a statistic
/organizations/aquaya/projects/water-quality/statistics/mean-pH?edit=true
: edit a statistic; accepts GET or POST
'''
user = User.objects(email=session['email'])[0]
orgs = Organization.objects(label=org_label)
if not orgs:
flash('Organization "%s" not found, sorry!' % org_label, 'warning')
return redirect(url_for('organizations'))
org = orgs[0]
# permission-check
if org not in user.organizations and not user.admin_rights:
app.logger.error('%s tried to view a project but was \
denied for want of admin rights' % session['email'])
abort(404)
# find the project
projects = Project.objects(label=project_label, organization=org)
if not projects:
flash('Project "%s" not found, sorry!' % project_label, 'warning')
return redirect(url_for('organizations', org_label=org.label))
project = projects[0]
if request.method == 'POST':
# we have a statistic_label
statistics = Statistic.objects(label=statistic_label, project=project)
if not statistics:
abort(404)
statistic = statistics[0]
form_type = request.form.get('form_type', '')
if form_type == 'info':
if statistic.name != request.form.get('name', ''):
name = request.form.get('name', '')
statistic.name = name
statistics = Statistic.objects(project=project).only('label')
labels = [s.label for s in statistics]
statistic.label = utilities.generate_label(name, labels)
statistic.description = request.form.get('description', '')
statistic.statistic_type = request.form.get('statistic_type', '')
# header is of the form 'header_id__4abcd0012'
header = request.form.get('header', '')
header_id = header.split('header_id__')[1]
header = Header.objects(id=header_id)[0]
statistic.header = header
elif form_type == 'create-table':
compute_multiple = request.form.get('compute_multiple', '')
if compute_multiple == 'true':
statistic.pivot = True
else:
statistic.pivot = False
header_id = request.form.get('compute_multiple_header', '')
headers = Header.objects(project=project, id=header_id)
if not headers:
abort(404)
statistic.pivot_header = headers[0]
# the name of the select with the values is based on header id
values = request.form.getlist('header_unique_values')
escaped_values = [bleach.clean(v) for v in values]
statistic.pivot_values = escaped_values
statistic.save()
flash('Table settings saved successfully', 'success')
return redirect(url_for('statistics'
, org_label=org.label
, project_label=project.label
, statistic_label=statistic.label
, edit='true'))
elif form_type == 'filters':
# extract the 'any filters' vs 'all' distinction
filter_settings = request.form.get('apply_any_filters', '')
if filter_settings == 'true':
statistic.apply_any_filters = True
else:
statistic.apply_any_filters = False
statistic.save()
# attach filter to statistic
requested_filter_ids = request.form.getlist('filters')
attached_filters = []
for requested_id in requested_filter_ids:
prefix, filter_id = requested_id.split('__')
filters = Filter.objects(id=filter_id)
if not filters:
abort(404)
attached_filters.append(filters[0])
statistic.update(set__filters = attached_filters)
return redirect(url_for('statistics', org_label=org.label
, project_label=project.label
, statistic_label=statistic.label, edit='true'))
elif form_type == 'admin':
# delete the statistic
name = statistic.name
utilities.delete_statistic(statistic, session['email'])
flash('statistic "%s" was deleted successfully' % name, 'success')
return redirect(url_for('statistics', org_label=org.label
, project_label=project.label))
else:
# bad 'form_type'
abort(404)
try:
statistic.save()
flash('Changes saved successfully', 'success')
return redirect(url_for('statistics', org_label=org.label
, project_label=project.label
, statistic_label=statistic.label, edit='true'))
except:
app.logger.error('%s experienced an error saving info about %s' % (
session['email'], request.form['name']))
flash('Error saving changes -- make sure statistic names are \
unique.', 'error')
return redirect(url_for('statistics', org_label=org.label
, project_label=project.label, statistic_label=statistic_label
, edit='true'))
if request.method == 'GET':
if statistic_label:
statistics = Statistic.objects(label=statistic_label
, project=project)
if not statistics:
app.logger.error('%s tried to access a statistic that does \
not exist' % session['email'])
flash('Statistic "%s" not found, sorry!' % statistic_label
, 'warning')
return redirect(url_for('projects', org_label=org.label
, project_label=project.label))
statistic = statistics[0]
if request.args.get('edit', '') == 'true':
# valid statistic types
statistic_types = constants.statistic_types
available_filters = Filter.objects(project=project)
# find all string-type headers
# these are available for compute_multiple
string_type_headers = []
for header in project.ordered_schema:
if header.display and header.data_type == 'string':
string_type_headers.append(header)
return render_template('statistic_edit.html'
, statistic = statistic
, statistic_types = statistic_types
, available_filters = available_filters
, string_type_headers = string_type_headers)
else:
# apply the filters and show matches
conditions = {
'project': project
, 'unique': True
, 'visible': True
}
entries = Entry.objects(**conditions)
# sort the entries based on project defaults
entries = utilities.sort_entries(project, entries)
project_count = len(entries)
for filter in statistic.filters:
if not filter.comparison:
flash('filter "%s" does not have a complete \
definition' % filter.name, 'warning')
return render_template('statistic.html'
, statistic=statistic
, entries=entries
, project_count=project_count)
# apply relevant filters and then computes the statistic
if statistic.pivot:
# returns dict keyed by the statistic's pivot_values
result = utilities.compute_pivot_statistic(statistic
, entries)
else:
# returns a single result
result = utilities.compute_statistic(statistic, entries)
# find matches to the applied filters
matches = utilities.apply_filters(statistic.filters, entries
, statistic.apply_any_filters)
total_matches = len(matches)
# pagination
entries_per_page = 10
pages = utilities.calculate_pages(len(matches)
, entries_per_page=entries_per_page)
# validate the requested page
current_page = utilities.validate_page_request(
request.args.get('page', 1), pages)
# manually paginate
start_index = entries_per_page * (current_page - 1)
end_index = start_index + entries_per_page
paginated_matches = matches[start_index:end_index]
return render_template('statistic.html'
, statistic=statistic
, entries=paginated_matches
, total_matches = total_matches
, project_count=project_count
, result=result
, current_page = current_page
, number_of_pages = pages)
if request.args.get('create', '') == 'true':
# create a new statistic
# CSRF validation
token = request.args.get('token', '')
if not verify_token(token):
abort(403)
try:
statistic_name = 'stat-%s' \
% utilities.generate_random_string(6)
new_statistic= Statistic(
creation_time = datetime.datetime.utcnow()
, creator = user
, label = statistic_name.lower()
, project = project
, name = statistic_name
)
new_statistic.save()
app.logger.info('statistic created by %s' % session['email'])
flash('statistic created; please change the defaults'
, 'success')
except:
app.logger.error('statistic creation failed for %s' % \
session['email'])
flash('There was an error, sorry :/', 'error')
return redirect(url_for('projects', org_label=org.label
, project=project.label))
# redirect to the editing screen
return redirect(url_for('statistics', org_label=org.label
, project_label=project.label
, statistic_label=new_statistic.label, edit='true'))
# no statistic in particular was specified; show em all
statistics = Statistic.objects(project=project)
# calculate the results of these statistics
statistic_results = {}
unique_entries = Entry.objects(project=project, unique=True
, visible=True)
for statistic in statistics:
statistic_results[statistic.name] = \
utilities.compute_statistic(statistic, unique_entries)
return render_template('project_statistics.html', project=project
, statistics=statistics, statistic_results=statistic_results)
def projects(org_label, project_label):
''' project wrangling
/organizations/aquaya/projects?create=true
: create a new project, immediately redirect to editing
/organizations/aquaya/projects/water-quality?edit=true
: edit a project; accepts GET or POST
/organizations/aquaya/projects/water-quality
: view a project
'''
user = User.objects(email=session['email'])[0]
orgs = Organization.objects(label=org_label)
if not orgs:
flash('Organization "%s" not found, sorry!' % org_label, 'warning')
return redirect(url_for('organizations'))
org = orgs[0]
# permission-check
if org not in user.organizations and not user.admin_rights:
app.logger.error('%s tried to view a project but was \
denied for want of admin rights' % session['email'])
abort(404)
if request.method == 'GET':
if not project_label and request.args.get('create', '') == 'true':
# create a new project
# CSRF validation
token = request.args.get('token', '')
if not verify_token(token):
abort(403)
try:
name = 'prj-%s' % utilities.generate_random_string(6)
new_project = Project(
creation_time = datetime.datetime.utcnow()
, label = name.lower()
, name = name
, organization = org
, update_time = datetime.datetime.utcnow()
)
new_project.save()
org.update(push__projects=new_project)
app.logger.info('project created by %s' % session['email'])
flash('project created; please change the defaults', 'success')
except:
app.logger.error('project creation failed for %s' % \
session['email'])
flash('There was an error in the form, sorry :/', 'error')
return redirect(url_for('organizations', org_label=org.label))
# redirect to the editing screen
return redirect(url_for('projects', org_label=org.label
, project_label=new_project.label, edit='true'))
elif not project_label:
# could show org's projects here but that's visible on the org page
abort(404)
else:
# we have a project label
projects = Project.objects(label=project_label, organization=org)
if not projects:
flash('Project "%s" not found, sorry!' % project_label
, 'warning')
return redirect(url_for('organizations', org_label=org.label))
project = projects[0]
if request.args.get('edit', '') == 'true':
# show the editor
return render_template('project_edit.html', project=project)
else:
upload_count = Upload.objects(project=project).count()
entry_count = Entry.objects(project=project, visible=True
, unique=True).count()
filter_count = Filter.objects(project=project).count()
graph_count = Graph.objects(project=project).count()
statistic_count = Statistic.objects(project=project).count()
report_count = Report.objects(project=project).count()
schedule_count = Schedule.objects(project=project).count()
connection_count = Connection.objects(project=project).count()
comment_count = Comment.objects(project=project).count()
if not entry_count:
flash('To get started, connect this project to a data \
or manually add some files.', 'info')
return render_template('project.html', project=project
, upload_count=upload_count
, entry_count=entry_count
, filter_count=filter_count
, graph_count=graph_count
, statistic_count=statistic_count
, report_count=report_count
, schedule_count=schedule_count
, connection_count=connection_count
, comment_count=comment_count)
elif request.method == 'POST':
# we have a project label
projects = Project.objects(label=project_label, organization=org)
if not projects:
abort(404)
project = projects[0]
form_type = request.form.get('form_type', '')
if form_type == 'info':
if project.name != request.form.get('name', ''):
name = request.form.get('name', '')
project.name = name
projects = Project.objects(organization=org).only('label')
labels = [p.label for p in projects]
project.label = utilities.generate_label(name, labels)
project.update_time = datetime.datetime.utcnow()
if project.description != request.form.get('description', ''):
app.logger.info('%s edited the description of %s to %s' % (
session['email'], project.name
, request.form.get('description', '')))
project.description = request.form.get('description', '')
project.update_time = datetime.datetime.utcnow()
elif form_type == 'admin':
# delete the project
# permission-check first
name = project.name
(upload_count, entry_count) = utilities.delete_project(project
, session['email'])
flash('Project "%s" was deleted successfully. %s uploads and \
%s entries were deleted as well' % (name, upload_count,
entry_count), 'success')
return redirect(url_for('organizations', org_label=org.label))
else:
# bad 'form_type'
abort(404)
try:
project.save()
flash('changes saved successfully', 'success')
except:
app.logger.error('%s experienced an error saving info about the \
project %s' % (session['email'], request.form['name']))
flash('error, make sure names are unique', 'error')
return redirect(url_for('organizations', org_label=org.label))
return redirect(url_for('projects', org_label=org.label
, project_label=project.label))