def connections(org_label, project_label, connection_label):
""" connection configuration management
/organizations/aquaya/projects/water-quality/connections
: viewing a list of this project's connections
/organizations/aquaya/projects/water-quality/connections?create=true
: create a new connection config, immediately redirect to editing
/organizations/aquaya/projects/water-quality/connections/ivrhub
: view the 'ivrhub' connection config
/organizations/aquaya/projects/water-quality/connections/ivrhub?edit=true
: edit a connection config; accepts GET or POST
"""
user = User.objects(email=session["email"])[0]
orgs = Organization.objects(label=org_label)
if not orgs:
flash('Organization "%s" not found, sorry!' % org_label, "warning")
return redirect(url_for("organizations"))
org = orgs[0]
# permission-check
if org not in user.organizations and not user.admin_rights:
app.logger.error(
"%s tried to view a project but was \
denied for want of admin rights"
% session["email"]
)
abort(404)
# find the project
projects = Project.objects(label=project_label, organization=org)
if not projects:
flash('Project "%s" not found, sorry!' % project_label, "warning")
return redirect(url_for("organizations", org_label=org.label))
project = projects[0]
if request.method == "POST":
# we have a connection_label
connections = Connection.objects(label=connection_label, project=project)
if not connections:
abort(404)
connection = connections[0]
form_type = request.form.get("form_type", "")
if form_type == "commcare_configuration":
connection.credentials = {
"username": request.form.get("username", ""),
"password": request.form.get("password", ""),
}
connection.domain = request.form.get("domain", "")
connection.export_tag = request.form.get("export_tag", "")
include_admin = request.form.get("include_admin", "")
if include_admin == "true":
connection.include_admin = True
else:
connection.include_admin = False
include_metadata = request.form.get("include_metadata", "")
if include_metadata == "true":
connection.include_metadata = True
else:
connection.include_metadata = False
connection.save()
flash("CommCare settings saved successfully.", "success")
url = url_for(
"connections",
org_label=org.label,
project_label=project.label,
connection_label=connection.label,
edit="true",
)
return redirect(url + "#commcare")
elif form_type == "schedule":
old_interval = connection.schedule.interval
new_interval = {
"type": request.form.get("schedule_type", ""),
"at": request.form.get("send_at", ""),
"on_day": request.form.get("send_on_day", ""),
}
if old_interval != new_interval:
repeating_task = connection.schedule
# update the interval
repeating_task.interval = new_interval
repeating_task.save()
# invalidate the running job and schedule the new one
update_scheduled_connection(connection)
flash("Connection interval saved successfully.", "success")
url = url_for(
"connections",
org_label=org.label,
project_label=project.label,
connection_label=connection.label,
edit="true",
)
return redirect(url + "#schedule")
elif form_type == "admin":
# delete the connection
# keeping the associated messages as they're an important log
name = connection.name
utilities.delete_connection(connection, session["email"])
# see if, given this deletion, the dupes are still dupes
unduped_count = utilities.recheck_duplicate_entries(project)
if unduped_count:
flash(
"%s entries were automatically added to the project \
as they are no longer duplicates"
% unduped_count,
"info",
)
flash('Connection "%s" was deleted successfully' % name, "success")
return redirect(url_for("connections", org_label=org.label, project_label=project.label))
else:
# bad 'form_type'
abort(404)
try:
connection.save()
flash("Changes saved successfully.", "success")
return redirect(
url_for(
"connections",
org_label=org.label,
project_label=project.label,
connection_label=connection.label,
edit="true",
)
)
except:
app.logger.error("%s experienced an error saving info about %s" % (session["email"], request.form["name"]))
flash(
"Error saving changes - make sure connection names are \
unique.",
"error",
)
return redirect(
url_for(
"connections",
org_label=org.label,
project_label=project.label,
connection_label=connection_label,
edit="true",
)
)
if request.method == "GET":
if connection_label:
connections = Connection.objects(label=connection_label, project=project)
if not connections:
app.logger.error(
"%s tried to access a connection that does \
not exist"
% session["email"]
)
flash('Connection "%s" not found, sorry!' % connection_label, "warning")
return redirect(url_for("connections", org_label=org.label, project_label=project.label))
connection = connections[0]
if request.args.get("edit", "") == "true":
current_time = datetime.datetime.utcnow()
return render_template("connection_edit.html", connection=connection, current_time=current_time)
elif request.args.get("fire", "") == "true":
if ready_to_connect(connection):
# enqueue the job immediately
redis_config = app.config["REDIS_CONFIG"]
use_connection(Redis(redis_config["host"], redis_config["port"], password=redis_config["password"]))
queue = Queue()
job = queue.enqueue_call(func=connect_to_source, args=(str(connection.id),), timeout=900)
flash(
"Connecting to data source, this may take a few \
moments. Check the connection log for updates.",
"info",
)
# save the id of this job
connection.schedule.update(set__next_task_id=job.id)
else:
flash("This connection is not properly configured.", "error")
return redirect(
url_for(
"connections",
org_label=org.label,
project_label=project.label,
connection_label=connection_label,
)
)
else:
connection_log = ConnectionLogEntry.objects(project=project).order_by("-completion_time")
return render_template("connection.html", connection=connection, connection_log=connection_log)
if request.args.get("create", "") == "true":
# create a new connection
# first create a repeating task object
new_repeater = RepeatingTask()
new_repeater.save()
service = request.args.get("service", "")
if service == "commcare":
# see if one already exists
cxns = CommCareConnection.objects(project=project)
if cxns:
flash(
"A data connection to CommCare already exists for \
this project. Create a new project if you want to \
connect to another CommCare source."
)
return redirect(url_for("connections", org_label=org.label, project_label=project.label))
# create a new one
new_connection = CommCareConnection(
description="A connection to data at commcarehq.org",
schedule=new_repeater,
label="commcare",
project=project,
name="CommCare",
service_type="commcare",
)
new_connection.save()
app.logger.info("CommCare connection created by %s" % session["email"])
flash(
"CommCare connection initialized. Please change the \
default values to complete the link.",
"info",
)
else:
flash("We are still working on connecting to that service.", "info")
return redirect(url_for("connections", org_label=org.label, project_label=project.label))
# redirect to the editing screen
return redirect(
url_for(
"connections",
org_label=org.label,
project_label=project.label,
connection_label=new_connection.label,
edit="true",
)
)
# no connection in particular was specified so show them all
cxns = CommCareConnection.objects(project=project)
commcare_connection = None
if cxns:
commcare_connection = cxns[0]
return render_template("project_connections.html", project=project, commcare_connection=commcare_connection)
def uploads(o_label, p_label, u_label):
''' uploadin
/organizations/aquaya/projects/water-quality/uploads?create=true
: create a new upload, immediately redirect to editing
/organizations/aquaya/projects/water-quality/uploads/may-2012
: view an upload
/organizations/aquaya/projects/water-quality/uploads/may-2012?edit=true
: edit an upload; accepts GET or POST
'''
# can't wrap the line through a route or spaces are injected
# hence this silliness:
org_label = o_label
project_label = p_label
upload_label = u_label
user = User.objects(email=session['email'])[0]
orgs = Organization.objects(label=org_label)
if not orgs:
flash('Organization "%s" not found, sorry!' % org_label, 'warning')
return redirect(url_for('organizations'))
org = orgs[0]
# permission-check
if org not in user.organizations and not user.admin_rights:
app.logger.error('%s tried to view a project but was \
denied for want of admin rights' % session['email'])
abort(404)
projects = Project.objects(label=project_label, organization=org)
if not projects:
flash('Project "%s" not found, sorry!' % project_label, 'warning')
return redirect(url_for('organizations', org_label=org.label))
project = projects[0]
if request.method == 'GET':
if not upload_label and request.args.get('create', '') == 'true':
# create a new upload
# CSRF validation
token = request.args.get('token', '')
if not verify_token(token):
abort(403)
try:
name = 'upl-%s' % utilities.generate_random_string(6)
new_upload = Upload(
label = name.lower()
, name = name
, project = project
)
new_upload.save()
project.update(push__uploads=new_upload)
app.logger.info('upload created by %s' % session['email'])
flash('please select a file and make other edits', 'warning')
except:
app.logger.error('upload creation failed for %s' % \
session['email'])
flash('There was an error, sorry :/', 'error')
return redirect(url_for('projects', org_label=org.label
, project_label=project.label))
# redirect to the editing screen
return redirect(url_for('uploads', o_label=org.label
, p_label=project.label, u_label=new_upload.label
, edit='true'))
elif not upload_label:
# could show projects uploads but punting for now
unique_entries = Entry.objects(project=project, unique=True
, visible=True)
duplicate_entries = Entry.objects(project=project
, unique=False)
hidden_entries = Entry.objects(project=project, visible=False
, unique=True)
upload_entry_counts = {}
for upload in project.uploads:
upload_entry_counts[upload.id] = Entry.objects(upload=upload).count()
return render_template('project_uploads.html', project=project
, unique_entries=unique_entries
, duplicate_entries=duplicate_entries
, upload_entry_counts=upload_entry_counts
, hidden_entries=hidden_entries)
else:
# we have an upload label
uploads = Upload.objects(label=upload_label, project=project)
if not uploads:
flash('Upload "%s" not found, sorry!' % upload_label
, 'warning')
return redirect(url_for('projects', org_label=org.label
, project_label=project.label))
upload = uploads[0]
if request.args.get('edit', '') == 'true':
# show the editor
# queue up any flashed messages
# these get set during the worker's processing of a file
worker_messages = upload.worker_messages
for message in worker_messages:
flash(message['message'], message['status'])
upload.update(set__worker_messages = [])
return render_template('upload_edit.html', upload=upload
, number_of_pages=0)
else:
if not upload.filename:
# need to add one so redirect to editing
flash('please specify a file for this upload', 'warning')
return redirect(url_for('uploads', o_label=org.label
, p_label=project.label, u_label=upload.label
, edit='true'))
else:
# things are still in progress
if not upload.s3_key:
flash('Your file is being processed in the '
'background. Refresh the page to see updates.'
, 'info')
return render_template('upload.html', upload=upload
, number_of_pages=0)
# otherwise upload is complete, show the data
# pagination
total_entries = Entry.objects(project=project
, upload=upload).count()
entries_per_page = 10
pages = utilities.calculate_pages(total_entries
, entries_per_page=entries_per_page)
# validate the requested page
current_page = utilities.validate_page_request(
request.args.get('page', 1), pages)
# get the sorted entries
entries = utilities.query_entries(project
, upload = upload
, skip = (entries_per_page * (current_page - 1))
, limit = entries_per_page)
hidden_entries_count = Entry.objects(upload=upload
, visible=False).count()
# queue up any flashed messages
# these get set during the worker's processing of a file
worker_messages = upload.worker_messages
for message in worker_messages:
flash(message['message'], message['status'])
upload.update(set__worker_messages = [])
return render_template('upload.html'
, upload=upload
, entries=entries
, total_entries = total_entries
, hidden_entries_count=hidden_entries_count
, current_page = current_page
, number_of_pages = pages)
elif request.method == 'POST':
# we have an upload label
uploads = Upload.objects(label=upload_label, project=project)
if not uploads:
abort(404)
upload = uploads[0]
form_type = request.form.get('form_type', '')
if form_type == 'info':
if upload.name != request.form.get('name', ''):
name = request.form.get('name', '')
upload.update(set__name = name)
uploads = Upload.objects(project=project).only('label')
labels = [u.label for u in uploads]
upload.update(set__label =
utilities.generate_label(name, labels))
upload.project.update(set__update_time =
datetime.datetime.utcnow())
upload.reload()
if upload.description != request.form.get('description', ''):
upload.update(set__description =
request.form.get('description', ''))
upload.project.update(set__update_time =
datetime.datetime.utcnow())
if request.files.get('data_file', ''):
data_file = request.files.get('data_file')
try:
filename = uploaded_data.save(data_file)
if '..' in filename or filename.startswith('/'):
app.logger.info('%s tried to upload a malicious file \
"%s"' % (session['email'], filename))
flash('bad filename, sorry :/', 'error')
return redirect(url_for('uploads'
, o_label=org.label, p_label=project.label
, u_label=upload.label, edit='true'))
absolute_filename = uploaded_data.path(filename)
except UploadNotAllowed:
app.logger.info('%s tried to upload an unsupported file'
% session['email'])
flash('we currently only support .xls files, sorry :/'
, 'error')
return redirect(url_for('uploads'
, o_label=org.label, p_label=project.label
, u_label=upload.label, edit='true'))
upload.update(set__filename =
os.path.basename(absolute_filename))
upload.update(set__uploaded_by = user)
# enqueue upload-processing
redis_config = app.config['REDIS_CONFIG']
use_connection(Redis(redis_config['host'], redis_config['port']
, password=redis_config['password']))
queue = Queue()
queue.enqueue(process_uploaded_file, upload.id,
absolute_filename)
elif form_type == 'admin':
# delete the upload
name = upload.name
# pull out the upload from the parent project first
project.update(pull__uploads=upload)
# delete associated entries, remove from system, remove from s3
deleted_entries_count = utilities.delete_upload(upload
, session['email'])
flash('upload "%s" was deleted successfully; %s entries deleted \
as well' % (name, deleted_entries_count), 'success')
# see if, given this deletion, the dupes are still dupes
unduped_count = utilities.recheck_duplicate_entries(project)
if unduped_count:
flash('%s entries were automatically added to the project \
as they are no longer duplicates' % unduped_count, 'info')
return redirect(url_for('uploads', o_label=org.label
, p_label=project.label))
else:
# bad 'form_type'
abort(404)
try:
upload.save()
project.save()
except:
app.logger.error('%s experienced an error saving info about the \
upload %s' % (session['email'], request.form['name']))
flash('error, make sure names are unique', 'error')
return redirect(url_for('projects'
, org_label=upload.project.organization.label
, project_label=upload.project.label))
return redirect(url_for('uploads'
, o_label=upload.project.organization.label, p_label=project.label
, u_label=upload.label))
