def get_config(self, session):
try:
limit = float(self.get_argument('limit', '0'))
wall_time = float(self.get_argument('wall_time', '0'))
except ValueError as e:
raise errors.ModelError(str(e))
max_wall_time = config.get_setting(session, 'custom_timeout') * 1000
if wall_time == 0:
wall_time = max_wall_time
elif not 0 <= wall_time <= max_wall_time:
raise errors.ModelError('Query wall time is out of bounds')
max_limit = config.get_setting(session, 'custom_max_limit')
if limit == 0:
limit = max_limit
elif not 0 <= limit <= max_limit:
raise errors.ModelError('Query row limit is out of bounds')
return DefaultMunch(
undefined, {
'wall_time': int(wall_time * 1000),
'limit': int(limit),
'base_url': config.get_setting(session, 'app_base_url'),
})
def paginate(self, query, optional=False):
if optional and self.get_argument("page", None) is None:
return query
page_size = self.get_argument("pageSize", str(Paginate.MAX_PAGE_SIZE))
try:
page_size = int(page_size)
except ValueError:
raise errors.ModelError("Invalid page size")
if page_size > Paginate.MAX_PAGE_SIZE:
raise errors.ModelError("Page size is too large (max %d)" %
Paginate.MAX_PAGE_SIZE)
page = self.get_argument("page", "0")
try:
page = int(page)
except ValueError:
raise errors.ModelError("Invalid page")
if page < 0:
raise errors.ModelError("Page must be non-negative")
query.distinct()
num_items = query.count()
self.set_header('Page-Count', "%d" % ceil(num_items / page_size))
self.set_header('Page-Index', "%d" % page)
self.set_header('Page-Item-Count', "%d" % page_size)
query = query.limit(page_size)
query = query.offset(page * page_size)
return query
def put(self, response_type_id):
'''Update existing'''
program_id = self.get_argument('programId', '')
with model.session_scope() as session:
user_session = self.get_user_session(session)
response_type = (session.query(model.ResponseType).get(
(response_type_id, program_id)))
if not response_type:
raise errors.MissingDocError("No such response type")
policy = user_session.policy.derive({
'program':
response_type.program,
'surveygroups':
response_type.program.surveygroups,
})
policy.verify('surveygroup_interact')
policy.verify('response_type_edit')
if 'name' in self.request_son:
rt_by_name = (session.query(model.ResponseType).filter(
model.ResponseType.program_id == program_id).filter(
model.ResponseType.name ==
self.request_son.name).first())
if rt_by_name and rt_by_name != response_type:
raise errors.ModelError(
"'" + self.request_son.name +
"' as a response type of that name already exists")
try:
self._update(response_type, self.request_son)
except ResponseTypeError as e:
raise errors.ModelError(str(e))
except voluptuous.error.Error as e:
raise errors.ModelError(str(e))
except Exception as e:
raise errors.ModelError(str(e))
verbs = []
# Check if modified now to avoid problems with autoflush later
if session.is_modified(response_type):
verbs.append('update')
calculator = Calculator.structural()
for measure in response_type.measures:
for qnode_measure in measure.qnode_measures:
calculator.mark_measure_dirty(qnode_measure)
calculator.execute()
act = Activities(session)
act.record(user_session.user, response_type, verbs)
act.ensure_subscription(user_session.user, response_type,
response_type.program, self.reason)
self.get(response_type_id)
def post(self, user_id):
'''
Create a new user.
'''
if user_id:
raise errors.MethodError("Can't use POST for existing users.")
try:
user_input_schema(self.request_son)
except voluptuous.error.Invalid as e:
raise errors.ModelError.from_voluptuous(e)
with model.session_scope() as session:
user_session = self.get_user_session(session)
org = (
session.query(model.Organisation)
.get(self.request_son['organisation']['id']))
if not org:
raise errors.ModelError("No such organisation")
user = model.AppUser(organisation=org)
try:
assign_surveygroups(user_session, user, self.request_son)
except ValueError as e:
raise errors.ModelError(str(e))
policy = user_session.policy.derive({
'org': user.organisation,
'user': user,
'target': self.request_son,
'surveygroups': user.surveygroups,
})
policy.verify('surveygroup_interact')
policy.verify('user_add')
policy.verify('user_change_role')
self.check_password(self.request_son.password)
self._update(user, self.request_son, session)
session.add(user)
# Need to flush so object has an ID to record action against.
session.flush()
act = Activities(session)
act.record(user_session.user, user, ['create'])
act.ensure_subscription(
user_session.user, user, user.organisation, self.reason)
act.subscribe(user, user.organisation)
self.reason("New user subscribed to organisation")
user_id = user.id
self.get(user_id)
def post(self, response_type_id):
'''Create new'''
if response_type_id:
raise errors.ModelError("Can't specify ID when creating")
program_id = self.get_argument('programId', '')
with model.session_scope() as session:
user_session = self.get_user_session(session)
program = session.query(model.Program).get(program_id)
if not program:
raise errors.MissingDocError("No such program")
policy = user_session.policy.derive({
'program':
program,
'surveygroups':
program.surveygroups,
})
policy.verify('surveygroup_interact')
policy.verify('response_type_add')
rt_by_name = (session.query(model.ResponseType).filter(
model.ResponseType.program_id == program_id).filter(
model.ResponseType.name == self.request_son.name).first())
if rt_by_name:
raise errors.ModelError(
"'" + self.request_son.name +
"' as a response type of that name already exists")
response_type = model.ResponseType(program=program)
session.add(response_type)
try:
self._update(response_type, self.request_son)
except ResponseTypeError as e:
raise errors.ModelError(str(e))
except voluptuous.error.Error as e:
raise errors.ModelError(str(e))
try:
session.flush()
except sqlalchemy.exc.IntegrityError as e:
raise errors.ModelError.from_sa(e)
response_type_id = str(response_type.id)
# No need for survey update: RT is not being used yet
act = Activities(session)
act.record(user_session.user, response_type, ['create'])
act.ensure_subscription(user_session.user, response_type,
response_type.program, self.reason)
self.get(response_type_id)
def put(self, activity_id):
with model.session_scope() as session:
user_session = self.get_user_session(session)
activity = (session.query(model.Activity).get(activity_id))
if not activity:
raise errors.MissingDocError("No such activity")
if 'surveygroups' in self.request_son:
try:
assign_surveygroups(user_session, activity,
self.request_son)
except ValueError as e:
raise errors.ModelError(str(e))
if activity.ob_type == 'organisation':
org = get_ob(activity.ob_type, activity.ob_ids)
policy = user_session.policy.derive({
'activity':
activity,
'org':
org,
'surveygroups':
org.surveygroups,
})
elif not activity.ob_type:
policy = user_session.policy.derive({
'activity':
activity,
'surveygroups':
activity.surveygroups,
})
else:
raise errors.ModelError("That activity can't be modified")
policy.verify('surveygroup_interact')
if 'sticky' in self.request_son:
if self.request_son.sticky != activity.sticky:
policy.verify('post_pin')
activity.sticky = self.request_son.sticky
if 'message' in self.request_son:
if self.request_son.message != activity.message:
policy.verify('post_edit')
activity.message = self.request_son.message
son = ActivityHandler.TO_SON(activity)
self.set_header("Content-Type", "application/json")
self.write(json_encode(son))
self.finish()
def query(self):
until_date = self.get_argument('until', '')
if until_date:
until_date = datetime.datetime.fromtimestamp(float(until_date))
else:
until_date = datetime.datetime.utcnow()
period = self.get_argument('period', '')
if period:
period = datetime.timedelta(seconds=float(period))
else:
period = datetime.timedelta(days=7)
if period.days > 31:
raise errors.ModelError("Time period is too large")
from_date = until_date - period
# Only show sticky elements when viewing current time period
offset = abs((until_date - datetime.datetime.utcnow()).total_seconds())
include_sticky = offset < period.total_seconds() / 2
son, details = yield self.fetch_activities(from_date, until_date,
include_sticky)
for i, message in enumerate(details):
self.add_header('Profiling', "%d %s" % (i, message))
self.set_header("Content-Type", "application/json")
self.write(json_encode(son))
self.finish()
def put(self, ob_type, subscription_id):
if ob_type:
raise errors.ModelError(
"Can't provide object type when updating a subscription")
with model.session_scope() as session:
user_session = self.get_user_session(session)
subscription = (session.query(
model.Subscription).get(subscription_id))
if not subscription:
raise errors.MissingDocError("No such subscription")
subscription.subscribed = self.request_son.get('subscribed', False)
ob = get_ob(session, subscription.ob_type, subscription.ob_refs)
policy = user_session.policy.derive({
'user':
subscription.user,
'survey':
self.get_survey(ob),
'submission':
self.get_submission(ob),
})
policy.verify('subscription_edit')
subscription_id = str(subscription.id)
self.get('', subscription_id)
def post(self, ob_type, object_ids):
object_ids = object_ids.split(',')
if not ob_type:
raise errors.ModelError(
"Object type required when creating a subscription")
with model.session_scope() as session:
user_session = self.get_user_session(session)
ob = get_ob(session, ob_type, object_ids)
if not ob:
raise errors.MissingDocError("No such object")
act = Activities(session)
subscription = act.subscribe(user_session.user, ob)
subscription.subscribed = self.request_son.get('subscribed', False)
policy = user_session.policy.derive({
'user':
subscription.user,
'survey':
self.get_survey(ob),
'submission':
self.get_submission(ob),
})
policy.verify('subscription_add')
session.flush()
subscription_id = str(subscription.id)
self.get('', subscription_id)
def from_parameters(cls, parameters):
try:
width = int(parameters.get('interval_num', 1))
except ValueError:
raise errors.ModelError("Invalid interval")
units = parameters.get('interval_unit', 'months')
if units == 'years':
if width < 1:
raise errors.ModelError("Interval must be at least one year")
elif units == 'months':
if width not in {1, 2, 3, 6}:
raise errors.ModelError("Interval must be 1, 2, 3 or 6 months")
else:
raise errors.ModelError("Unrecognised interval %s" % units)
return cls(width, units)
def post(self, query_id, file_type):
parameters = self.request_son.copy()
with model.session_scope() as session:
custom_query = session.query(model.CustomQuery).get(query_id)
if not custom_query:
raise errors.MissingDocError("No such query")
user_session = self.get_user_session(session)
policy = user_session.policy.derive({
'custom_query': custom_query,
})
policy.verify('custom_query_execute')
if not custom_query.text:
raise errors.ModelError("Query is empty")
conf = self.get_config(session)
session.expunge(custom_query)
# Overwrite stored query text if query is parameterised
if parameters.text:
custom_query.text = parameters.text
yield self.export(custom_query, conf, file_type)
self.finish()
def background_task(self, title, description, surveygroup_ids):
with tempfile.NamedTemporaryFile() as fd:
fileinfo = self.request.files['file'][0]
fd.write(fileinfo['body'])
all_rows = self.read_sheet(fd.name)
with model.session_scope() as session:
user_session = self.get_user_session(session)
program = model.Program()
program.title = title
program.description = bleach.clean(description, strip=True)
session.add(program)
surveygroups = (session.query(model.SurveyGroup).filter(
model.SurveyGroup.id.in_(surveygroup_ids)).all())
if not len(surveygroups) == len(surveygroup_ids):
raise errors.ModelError("Some surveygroups could not be found")
program.surveygroups = set(surveygroups)
session.flush()
program_id = program.id
policy = user_session.policy.derive({
'program':
program,
'surveygroups':
program.surveygroups,
})
policy.verify('surveygroup_interact')
policy.verify('program_add')
self.process_structure_file(all_rows, session, program)
return program_id
def apply_config(self, session, conf):
log.debug("Setting wall time to %d" % conf.wall_time)
try:
session.execute("SET statement_timeout TO :wall_time",
{'wall_time': conf.wall_time})
except sqlalchemy.exc.SQLAlchemyError as e:
raise errors.ModelError("Failed to prepare database session: %s" %
e)
def _update(self, survey, son):
update = updater(survey, error_factory=errors.ModelError)
update('title', son)
update('description', son, sanitise=True)
try:
update('structure', son)
except voluptuous.Error as e:
raise errors.ModelError("Structure is invalid: %s" % str(e))
def check_concurrent_write(self, custom_query):
modified = self.request_son.get("latest_modified", 0)
# Convert to int to avoid string conversion errors during
# JSON marshalling.
if int(modified) < int(custom_query.modified.timestamp()):
raise errors.ModelError(
"This query has changed since you loaded the"
" page. Please copy or remember your changes and"
" refresh the page.")
def _update(self, submission, son):
update = updater(submission, error_factory=errors.ModelError)
update('title', son)
if son["created"]:
try:
created = datetime.datetime.fromtimestamp(son['created'])
except TypeError as e:
raise errors.ModelError("Invalid date")
update('created', {"created": created})
def background_task(self, program_id, survey_id, organisation_id, title):
with tempfile.NamedTemporaryFile() as fd:
fileinfo = self.request.files['file'][0]
fd.write(fileinfo['body'])
all_rows = self.read_sheet(fd.name)
with model.session_scope() as session:
user_session = self.get_user_session(session)
org = session.query(model.Organisation).get(organisation_id)
if not org:
raise errors.ModelError("No such organisation")
survey = (session.query(model.Survey).get((survey_id, program_id)))
if not survey:
raise errors.ModelError("No such survey")
submission = model.Submission()
submission.program = survey.program
submission.survey = survey
submission.organisation = org
submission.title = title
submission.approval = 'draft'
session.add(submission)
session.flush()
submission_id = submission.id
policy = user_session.policy.derive({
'org':
org,
'survey':
survey,
'surveygroups':
submission.surveygroups,
})
policy.verify('surveygroup_interact')
policy.verify('submission_add')
self.process_submission_file(all_rows, session, submission,
user_session.user)
return submission_id
def query(self, submission_id):
'''Get a list.'''
qnode_id = self.get_argument('qnodeId', '')
if not qnode_id:
raise errors.ModelError("qnode ID required")
with model.session_scope() as session:
user_session = self.get_user_session(session)
submission = (session.query(model.Submission).options(
joinedload('organisation')).filter_by(
id=submission_id).first())
if not submission:
raise errors.MissingDocError("No such submission")
policy = user_session.policy.derive({
'org':
submission.organisation,
'submission':
submission,
'surveygroups':
submission.surveygroups,
})
policy.verify('surveygroup_interact')
policy.verify('response_view')
rnode = (session.query(model.ResponseNode).get(
(submission_id, qnode_id)))
if not rnode:
responses = []
else:
responses = rnode.responses
to_son = ToSon(
# Fields to match from any visited object
r'/id$',
r'/score$',
r'/approval$',
r'/modified$',
r'/not_relevant$',
r'^/[0-9]+/error$',
# Descend into nested objects
r'/[0-9]+$',
r'/measure$',
)
if user_session.user.role == 'clerk':
to_son.exclude(r'/score$')
sons = to_son(responses)
self.set_header("Content-Type", "application/json")
self.write(json_encode(sons))
self.finish()
def post(self, survey_id):
'''Create new.'''
if survey_id:
raise errors.MethodError("Can't use POST for existing object")
program_id = self.get_argument('programId', '')
with model.session_scope() as session:
user_session = self.get_user_session(session)
program = (session.query(model.Program).options(
joinedload('surveygroups')).get(program_id))
if not program:
raise errors.ModelError("No such program")
## set only last level indexing_form 1 if no value, remove other level indexing_form
levels = len(self.request_son.structure.levels)
for i, l in enumerate(self.request_son.structure.levels):
if i + 1 != levels and l.indexing_from:
del l.indexing_from
if i + 1 == levels and (
(not l.indexing_from and l.indexing_from != 0) or
(not isinstance(l.indexing_from, Number))
or l.indexing_from < 0):
l.indexing_from = 1
survey = model.Survey(program=program)
self._update(survey, self.request_son)
session.add(survey)
# Need to flush so object has an ID to record action against.
session.flush()
policy = user_session.policy.derive({
'program':
program,
'survey':
survey,
'surveygroups':
program.surveygroups,
})
policy.verify('surveygroup_interact')
policy.verify('survey_add')
act = Activities(session)
act.record(user_session.user, survey, ['create'])
act.ensure_subscription(user_session.user, survey, survey.program,
self.reason)
survey_id = str(survey.id)
self.get(survey_id)
def request_son(self):
try:
return self._request_son
except AttributeError:
pass
try:
self._request_son = denormalise(json_decode(self.request.body))
except (TypeError, UnicodeError, ValueError) as e:
raise errors.ModelError(
"Could not decode request body: %s. Body started with %s" %
(str(e), self.request.body[0:30]))
return self._request_son
def put(self, organisation_id):
'''
Update an existing organisation.
'''
if not organisation_id:
raise errors.MethodError(
"Can't use PUT for new organisations (no ID).")
with model.session_scope() as session:
user_session = self.get_user_session(session)
org = session.query(model.Organisation).get(organisation_id)
if not org:
raise errors.MissingDocError("No such organisation")
try:
groups_changed = assign_surveygroups(user_session, org,
self.request_son)
except ValueError as e:
raise errors.ModelError(str(e))
policy = user_session.policy.derive({
'org':
org,
'surveygroups':
org.surveygroups,
})
policy.verify('org_edit')
# Check that user shares a common surveygroup with this org.
# Admins need permission to edit orgs outside their surveygroups
# though.
if not policy.check('admin'):
policy.verify('surveygroup_interact')
old_locations = list(org.locations)
self._update(org, self.request_son)
verbs = []
if (session.is_modified(org) or org.locations != old_locations
or groups_changed or session.is_modified(org.meta)):
verbs.append('update')
if org.deleted:
org.deleted = False
verbs.append('undelete')
act = Activities(session)
act.record(user_session.user, org, verbs)
act.ensure_subscription(user_session.user, org, org, self.reason)
self.get(organisation_id)
def get_parameters(self):
parameters = self.request_son.copy()
try:
parameters['min_constituents'] = int(
parameters.get('min_constituents', MIN_CONSITUENTS))
except ValueError:
raise errors.ModelError("Invalid minimum number of constituents")
if 'approval' not in parameters:
raise errors.ModelError("Approval status required")
interval = Interval.from_parameters(parameters)
parameters.min_date = interval.lower_bound(
datetime.datetime.utcfromtimestamp(parameters.get('min_date')))
parameters.max_date = interval.upper_bound(
datetime.datetime.utcfromtimestamp(parameters.get('max_date')))
self.reason("Date range: %s - %s" %
(parameters.min_date.strftime('%d %b %Y'),
parameters.max_date.strftime('%d %b %Y')))
return parameters
def get(self):
program_id_a = self.get_argument("programId1", '')
program_id_b = self.get_argument("programId2", '')
survey_id = self.get_argument("surveyId", '')
ignore_tags = set().union(self.get_arguments("ignoreTag"))
if not program_id_a:
raise errors.ModelError("Program ID 1 required")
if not program_id_b:
raise errors.ModelError("Program ID 2 required")
if not survey_id:
raise errors.ModelError("Survey ID required")
son, details = yield self.background_task(program_id_a, program_id_b,
survey_id, ignore_tags)
for i, message in enumerate(details):
self.add_header('Profiling', "%d %s" % (i, message))
self.set_header("Content-Type", "application/json")
self.write(json_encode(son))
self.finish()
def put(self, submission_id, measure_id, submeasure_id):
son = self.request_son
externals = son["externals"]
with model.session_scope() as session:
user_session = self.get_user_session(session)
response = (
session.query(model.Response)
.get((submission_id, measure_id)))
if response is None:
raise errors.MissingDocError("No such response")
org = response.submission.organisation
policy = user_session.policy.derive({
'org': org,
'surveygroups': org.surveygroups,
})
policy.verify('surveygroup_interact')
policy.verify('attachment_add')
for external in externals:
url = external.get('url', '').strip()
file_name = external.get('file_name', '').strip()
if url == '' and file_name == '':
continue
if url == '':
raise errors.ModelError(
"URL required for link '%s'" % file_name)
if measure_id!=submeasure_id:
attachment = model.Attachment(
organisation=response.submission.organisation,
response=response,
url=url,
file_name=file_name,
submeasure_id=submeasure_id,
storage='external'
)
else:
attachment = model.Attachment(
organisation=response.submission.organisation,
response=response,
url=url,
file_name=file_name,
storage='external'
)
session.add(attachment)
self.get(submission_id, measure_id, submeasure_id)
def post(self, file_type):
with model.session_scope() as session:
user_session = self.get_user_session(session)
policy = user_session.policy.derive({})
policy.verify('custom_query_preview')
text = to_basestring(self.request.body)
if not text:
raise errors.ModelError("Query is empty")
custom_query = model.CustomQuery(description="Preview", text=text)
conf = self.get_config(session)
yield self.export(custom_query, conf, file_type)
self.finish()
def get_version(self, session, response, version):
if not version:
return None
try:
version = int(version)
except ValueError:
raise errors.ModelError("Invalid version number")
if version == response.version:
return None
history = (session.query(model.ResponseHistory).get(
(response.submission_id, response.measure_id, version)))
if history is None:
raise errors.MissingDocError("No such version")
return history
def _update(self, user, son, session):
'''
Apply user-provided data to the saved model.
'''
update = updater(user, error_factory=errors.ModelError)
update('email', son)
update('email_interval', son)
update('name', son)
update('role', son)
update('password', son)
org = (
session.query(model.Organisation)
.get(son.organisation.id))
if not org:
raise errors.ModelError("No such organisation")
user.organisation = org
def get_version(self, session, custom_query, version):
if not version:
return None
try:
version = int(version)
except ValueError:
raise errors.ModelError("Invalid version number")
if version == custom_query.version:
return None
history = (session.query(model.CustomQueryHistory).get(
(custom_query.id, version)))
if history is None:
raise errors.MissingDocError("No such version")
return history
def check_approval(self, session, submission, approval):
approval_set = self.approval_set(approval)
n_relevant_responses = (session.query(model.Response).filter(
model.Response.submission_id == submission.id,
model.Response.approval.in_(approval_set)).count())
n_measures = (session.query(model.QnodeMeasure.measure_id).join(
model.QuestionNode).filter(
model.QuestionNode.survey_id == submission.survey_id,
model.QuestionNode.program_id == submission.program_id,
model.QnodeMeasure.program_id == submission.program_id,
model.QnodeMeasure.qnode_id == model.QuestionNode.id,
model.QuestionNode.deleted == False).distinct().count())
if n_relevant_responses < n_measures:
raise errors.ModelError(
"%d of %d responses are incomplete" %
(n_measures - n_relevant_responses, n_measures))
def post(self):
'''
Check the strength of a password.
'''
if 'password' not in self.request_son:
raise errors.ModelError("Please specify a password")
strength, threshold, improvements = test_password(
self.request_son['password'])
son = {
'threshold': threshold,
'strength': strength,
'improvements': improvements
}
self.set_header("Content-Type", "application/json")
self.write(json_encode(son))
self.finish()
