def get_config(self, session): try: limit = float(self.get_argument('limit', '0')) wall_time = float(self.get_argument('wall_time', '0')) except ValueError as e: raise errors.ModelError(str(e)) max_wall_time = config.get_setting(session, 'custom_timeout') * 1000 if wall_time == 0: wall_time = max_wall_time elif not 0 <= wall_time <= max_wall_time: raise errors.ModelError('Query wall time is out of bounds') max_limit = config.get_setting(session, 'custom_max_limit') if limit == 0: limit = max_limit elif not 0 <= limit <= max_limit: raise errors.ModelError('Query row limit is out of bounds') return DefaultMunch( undefined, { 'wall_time': int(wall_time * 1000), 'limit': int(limit), 'base_url': config.get_setting(session, 'app_base_url'), })
def paginate(self, query, optional=False): if optional and self.get_argument("page", None) is None: return query page_size = self.get_argument("pageSize", str(Paginate.MAX_PAGE_SIZE)) try: page_size = int(page_size) except ValueError: raise errors.ModelError("Invalid page size") if page_size > Paginate.MAX_PAGE_SIZE: raise errors.ModelError("Page size is too large (max %d)" % Paginate.MAX_PAGE_SIZE) page = self.get_argument("page", "0") try: page = int(page) except ValueError: raise errors.ModelError("Invalid page") if page < 0: raise errors.ModelError("Page must be non-negative") query.distinct() num_items = query.count() self.set_header('Page-Count', "%d" % ceil(num_items / page_size)) self.set_header('Page-Index', "%d" % page) self.set_header('Page-Item-Count', "%d" % page_size) query = query.limit(page_size) query = query.offset(page * page_size) return query
def put(self, response_type_id): '''Update existing''' program_id = self.get_argument('programId', '') with model.session_scope() as session: user_session = self.get_user_session(session) response_type = (session.query(model.ResponseType).get( (response_type_id, program_id))) if not response_type: raise errors.MissingDocError("No such response type") policy = user_session.policy.derive({ 'program': response_type.program, 'surveygroups': response_type.program.surveygroups, }) policy.verify('surveygroup_interact') policy.verify('response_type_edit') if 'name' in self.request_son: rt_by_name = (session.query(model.ResponseType).filter( model.ResponseType.program_id == program_id).filter( model.ResponseType.name == self.request_son.name).first()) if rt_by_name and rt_by_name != response_type: raise errors.ModelError( "'" + self.request_son.name + "' as a response type of that name already exists") try: self._update(response_type, self.request_son) except ResponseTypeError as e: raise errors.ModelError(str(e)) except voluptuous.error.Error as e: raise errors.ModelError(str(e)) except Exception as e: raise errors.ModelError(str(e)) verbs = [] # Check if modified now to avoid problems with autoflush later if session.is_modified(response_type): verbs.append('update') calculator = Calculator.structural() for measure in response_type.measures: for qnode_measure in measure.qnode_measures: calculator.mark_measure_dirty(qnode_measure) calculator.execute() act = Activities(session) act.record(user_session.user, response_type, verbs) act.ensure_subscription(user_session.user, response_type, response_type.program, self.reason) self.get(response_type_id)
def post(self, user_id): ''' Create a new user. ''' if user_id: raise errors.MethodError("Can't use POST for existing users.") try: user_input_schema(self.request_son) except voluptuous.error.Invalid as e: raise errors.ModelError.from_voluptuous(e) with model.session_scope() as session: user_session = self.get_user_session(session) org = ( session.query(model.Organisation) .get(self.request_son['organisation']['id'])) if not org: raise errors.ModelError("No such organisation") user = model.AppUser(organisation=org) try: assign_surveygroups(user_session, user, self.request_son) except ValueError as e: raise errors.ModelError(str(e)) policy = user_session.policy.derive({ 'org': user.organisation, 'user': user, 'target': self.request_son, 'surveygroups': user.surveygroups, }) policy.verify('surveygroup_interact') policy.verify('user_add') policy.verify('user_change_role') self.check_password(self.request_son.password) self._update(user, self.request_son, session) session.add(user) # Need to flush so object has an ID to record action against. session.flush() act = Activities(session) act.record(user_session.user, user, ['create']) act.ensure_subscription( user_session.user, user, user.organisation, self.reason) act.subscribe(user, user.organisation) self.reason("New user subscribed to organisation") user_id = user.id self.get(user_id)
def post(self, response_type_id): '''Create new''' if response_type_id: raise errors.ModelError("Can't specify ID when creating") program_id = self.get_argument('programId', '') with model.session_scope() as session: user_session = self.get_user_session(session) program = session.query(model.Program).get(program_id) if not program: raise errors.MissingDocError("No such program") policy = user_session.policy.derive({ 'program': program, 'surveygroups': program.surveygroups, }) policy.verify('surveygroup_interact') policy.verify('response_type_add') rt_by_name = (session.query(model.ResponseType).filter( model.ResponseType.program_id == program_id).filter( model.ResponseType.name == self.request_son.name).first()) if rt_by_name: raise errors.ModelError( "'" + self.request_son.name + "' as a response type of that name already exists") response_type = model.ResponseType(program=program) session.add(response_type) try: self._update(response_type, self.request_son) except ResponseTypeError as e: raise errors.ModelError(str(e)) except voluptuous.error.Error as e: raise errors.ModelError(str(e)) try: session.flush() except sqlalchemy.exc.IntegrityError as e: raise errors.ModelError.from_sa(e) response_type_id = str(response_type.id) # No need for survey update: RT is not being used yet act = Activities(session) act.record(user_session.user, response_type, ['create']) act.ensure_subscription(user_session.user, response_type, response_type.program, self.reason) self.get(response_type_id)
def put(self, activity_id): with model.session_scope() as session: user_session = self.get_user_session(session) activity = (session.query(model.Activity).get(activity_id)) if not activity: raise errors.MissingDocError("No such activity") if 'surveygroups' in self.request_son: try: assign_surveygroups(user_session, activity, self.request_son) except ValueError as e: raise errors.ModelError(str(e)) if activity.ob_type == 'organisation': org = get_ob(activity.ob_type, activity.ob_ids) policy = user_session.policy.derive({ 'activity': activity, 'org': org, 'surveygroups': org.surveygroups, }) elif not activity.ob_type: policy = user_session.policy.derive({ 'activity': activity, 'surveygroups': activity.surveygroups, }) else: raise errors.ModelError("That activity can't be modified") policy.verify('surveygroup_interact') if 'sticky' in self.request_son: if self.request_son.sticky != activity.sticky: policy.verify('post_pin') activity.sticky = self.request_son.sticky if 'message' in self.request_son: if self.request_son.message != activity.message: policy.verify('post_edit') activity.message = self.request_son.message son = ActivityHandler.TO_SON(activity) self.set_header("Content-Type", "application/json") self.write(json_encode(son)) self.finish()
def query(self): until_date = self.get_argument('until', '') if until_date: until_date = datetime.datetime.fromtimestamp(float(until_date)) else: until_date = datetime.datetime.utcnow() period = self.get_argument('period', '') if period: period = datetime.timedelta(seconds=float(period)) else: period = datetime.timedelta(days=7) if period.days > 31: raise errors.ModelError("Time period is too large") from_date = until_date - period # Only show sticky elements when viewing current time period offset = abs((until_date - datetime.datetime.utcnow()).total_seconds()) include_sticky = offset < period.total_seconds() / 2 son, details = yield self.fetch_activities(from_date, until_date, include_sticky) for i, message in enumerate(details): self.add_header('Profiling', "%d %s" % (i, message)) self.set_header("Content-Type", "application/json") self.write(json_encode(son)) self.finish()
def put(self, ob_type, subscription_id): if ob_type: raise errors.ModelError( "Can't provide object type when updating a subscription") with model.session_scope() as session: user_session = self.get_user_session(session) subscription = (session.query( model.Subscription).get(subscription_id)) if not subscription: raise errors.MissingDocError("No such subscription") subscription.subscribed = self.request_son.get('subscribed', False) ob = get_ob(session, subscription.ob_type, subscription.ob_refs) policy = user_session.policy.derive({ 'user': subscription.user, 'survey': self.get_survey(ob), 'submission': self.get_submission(ob), }) policy.verify('subscription_edit') subscription_id = str(subscription.id) self.get('', subscription_id)
def post(self, ob_type, object_ids): object_ids = object_ids.split(',') if not ob_type: raise errors.ModelError( "Object type required when creating a subscription") with model.session_scope() as session: user_session = self.get_user_session(session) ob = get_ob(session, ob_type, object_ids) if not ob: raise errors.MissingDocError("No such object") act = Activities(session) subscription = act.subscribe(user_session.user, ob) subscription.subscribed = self.request_son.get('subscribed', False) policy = user_session.policy.derive({ 'user': subscription.user, 'survey': self.get_survey(ob), 'submission': self.get_submission(ob), }) policy.verify('subscription_add') session.flush() subscription_id = str(subscription.id) self.get('', subscription_id)
def from_parameters(cls, parameters): try: width = int(parameters.get('interval_num', 1)) except ValueError: raise errors.ModelError("Invalid interval") units = parameters.get('interval_unit', 'months') if units == 'years': if width < 1: raise errors.ModelError("Interval must be at least one year") elif units == 'months': if width not in {1, 2, 3, 6}: raise errors.ModelError("Interval must be 1, 2, 3 or 6 months") else: raise errors.ModelError("Unrecognised interval %s" % units) return cls(width, units)
def post(self, query_id, file_type): parameters = self.request_son.copy() with model.session_scope() as session: custom_query = session.query(model.CustomQuery).get(query_id) if not custom_query: raise errors.MissingDocError("No such query") user_session = self.get_user_session(session) policy = user_session.policy.derive({ 'custom_query': custom_query, }) policy.verify('custom_query_execute') if not custom_query.text: raise errors.ModelError("Query is empty") conf = self.get_config(session) session.expunge(custom_query) # Overwrite stored query text if query is parameterised if parameters.text: custom_query.text = parameters.text yield self.export(custom_query, conf, file_type) self.finish()
def background_task(self, title, description, surveygroup_ids): with tempfile.NamedTemporaryFile() as fd: fileinfo = self.request.files['file'][0] fd.write(fileinfo['body']) all_rows = self.read_sheet(fd.name) with model.session_scope() as session: user_session = self.get_user_session(session) program = model.Program() program.title = title program.description = bleach.clean(description, strip=True) session.add(program) surveygroups = (session.query(model.SurveyGroup).filter( model.SurveyGroup.id.in_(surveygroup_ids)).all()) if not len(surveygroups) == len(surveygroup_ids): raise errors.ModelError("Some surveygroups could not be found") program.surveygroups = set(surveygroups) session.flush() program_id = program.id policy = user_session.policy.derive({ 'program': program, 'surveygroups': program.surveygroups, }) policy.verify('surveygroup_interact') policy.verify('program_add') self.process_structure_file(all_rows, session, program) return program_id
def apply_config(self, session, conf): log.debug("Setting wall time to %d" % conf.wall_time) try: session.execute("SET statement_timeout TO :wall_time", {'wall_time': conf.wall_time}) except sqlalchemy.exc.SQLAlchemyError as e: raise errors.ModelError("Failed to prepare database session: %s" % e)
def _update(self, survey, son): update = updater(survey, error_factory=errors.ModelError) update('title', son) update('description', son, sanitise=True) try: update('structure', son) except voluptuous.Error as e: raise errors.ModelError("Structure is invalid: %s" % str(e))
def check_concurrent_write(self, custom_query): modified = self.request_son.get("latest_modified", 0) # Convert to int to avoid string conversion errors during # JSON marshalling. if int(modified) < int(custom_query.modified.timestamp()): raise errors.ModelError( "This query has changed since you loaded the" " page. Please copy or remember your changes and" " refresh the page.")
def _update(self, submission, son): update = updater(submission, error_factory=errors.ModelError) update('title', son) if son["created"]: try: created = datetime.datetime.fromtimestamp(son['created']) except TypeError as e: raise errors.ModelError("Invalid date") update('created', {"created": created})
def background_task(self, program_id, survey_id, organisation_id, title): with tempfile.NamedTemporaryFile() as fd: fileinfo = self.request.files['file'][0] fd.write(fileinfo['body']) all_rows = self.read_sheet(fd.name) with model.session_scope() as session: user_session = self.get_user_session(session) org = session.query(model.Organisation).get(organisation_id) if not org: raise errors.ModelError("No such organisation") survey = (session.query(model.Survey).get((survey_id, program_id))) if not survey: raise errors.ModelError("No such survey") submission = model.Submission() submission.program = survey.program submission.survey = survey submission.organisation = org submission.title = title submission.approval = 'draft' session.add(submission) session.flush() submission_id = submission.id policy = user_session.policy.derive({ 'org': org, 'survey': survey, 'surveygroups': submission.surveygroups, }) policy.verify('surveygroup_interact') policy.verify('submission_add') self.process_submission_file(all_rows, session, submission, user_session.user) return submission_id
def query(self, submission_id): '''Get a list.''' qnode_id = self.get_argument('qnodeId', '') if not qnode_id: raise errors.ModelError("qnode ID required") with model.session_scope() as session: user_session = self.get_user_session(session) submission = (session.query(model.Submission).options( joinedload('organisation')).filter_by( id=submission_id).first()) if not submission: raise errors.MissingDocError("No such submission") policy = user_session.policy.derive({ 'org': submission.organisation, 'submission': submission, 'surveygroups': submission.surveygroups, }) policy.verify('surveygroup_interact') policy.verify('response_view') rnode = (session.query(model.ResponseNode).get( (submission_id, qnode_id))) if not rnode: responses = [] else: responses = rnode.responses to_son = ToSon( # Fields to match from any visited object r'/id$', r'/score$', r'/approval$', r'/modified$', r'/not_relevant$', r'^/[0-9]+/error$', # Descend into nested objects r'/[0-9]+$', r'/measure$', ) if user_session.user.role == 'clerk': to_son.exclude(r'/score$') sons = to_son(responses) self.set_header("Content-Type", "application/json") self.write(json_encode(sons)) self.finish()
def post(self, survey_id): '''Create new.''' if survey_id: raise errors.MethodError("Can't use POST for existing object") program_id = self.get_argument('programId', '') with model.session_scope() as session: user_session = self.get_user_session(session) program = (session.query(model.Program).options( joinedload('surveygroups')).get(program_id)) if not program: raise errors.ModelError("No such program") ## set only last level indexing_form 1 if no value, remove other level indexing_form levels = len(self.request_son.structure.levels) for i, l in enumerate(self.request_son.structure.levels): if i + 1 != levels and l.indexing_from: del l.indexing_from if i + 1 == levels and ( (not l.indexing_from and l.indexing_from != 0) or (not isinstance(l.indexing_from, Number)) or l.indexing_from < 0): l.indexing_from = 1 survey = model.Survey(program=program) self._update(survey, self.request_son) session.add(survey) # Need to flush so object has an ID to record action against. session.flush() policy = user_session.policy.derive({ 'program': program, 'survey': survey, 'surveygroups': program.surveygroups, }) policy.verify('surveygroup_interact') policy.verify('survey_add') act = Activities(session) act.record(user_session.user, survey, ['create']) act.ensure_subscription(user_session.user, survey, survey.program, self.reason) survey_id = str(survey.id) self.get(survey_id)
def request_son(self): try: return self._request_son except AttributeError: pass try: self._request_son = denormalise(json_decode(self.request.body)) except (TypeError, UnicodeError, ValueError) as e: raise errors.ModelError( "Could not decode request body: %s. Body started with %s" % (str(e), self.request.body[0:30])) return self._request_son
def put(self, organisation_id): ''' Update an existing organisation. ''' if not organisation_id: raise errors.MethodError( "Can't use PUT for new organisations (no ID).") with model.session_scope() as session: user_session = self.get_user_session(session) org = session.query(model.Organisation).get(organisation_id) if not org: raise errors.MissingDocError("No such organisation") try: groups_changed = assign_surveygroups(user_session, org, self.request_son) except ValueError as e: raise errors.ModelError(str(e)) policy = user_session.policy.derive({ 'org': org, 'surveygroups': org.surveygroups, }) policy.verify('org_edit') # Check that user shares a common surveygroup with this org. # Admins need permission to edit orgs outside their surveygroups # though. if not policy.check('admin'): policy.verify('surveygroup_interact') old_locations = list(org.locations) self._update(org, self.request_son) verbs = [] if (session.is_modified(org) or org.locations != old_locations or groups_changed or session.is_modified(org.meta)): verbs.append('update') if org.deleted: org.deleted = False verbs.append('undelete') act = Activities(session) act.record(user_session.user, org, verbs) act.ensure_subscription(user_session.user, org, org, self.reason) self.get(organisation_id)
def get_parameters(self): parameters = self.request_son.copy() try: parameters['min_constituents'] = int( parameters.get('min_constituents', MIN_CONSITUENTS)) except ValueError: raise errors.ModelError("Invalid minimum number of constituents") if 'approval' not in parameters: raise errors.ModelError("Approval status required") interval = Interval.from_parameters(parameters) parameters.min_date = interval.lower_bound( datetime.datetime.utcfromtimestamp(parameters.get('min_date'))) parameters.max_date = interval.upper_bound( datetime.datetime.utcfromtimestamp(parameters.get('max_date'))) self.reason("Date range: %s - %s" % (parameters.min_date.strftime('%d %b %Y'), parameters.max_date.strftime('%d %b %Y'))) return parameters
def get(self): program_id_a = self.get_argument("programId1", '') program_id_b = self.get_argument("programId2", '') survey_id = self.get_argument("surveyId", '') ignore_tags = set().union(self.get_arguments("ignoreTag")) if not program_id_a: raise errors.ModelError("Program ID 1 required") if not program_id_b: raise errors.ModelError("Program ID 2 required") if not survey_id: raise errors.ModelError("Survey ID required") son, details = yield self.background_task(program_id_a, program_id_b, survey_id, ignore_tags) for i, message in enumerate(details): self.add_header('Profiling', "%d %s" % (i, message)) self.set_header("Content-Type", "application/json") self.write(json_encode(son)) self.finish()
def put(self, submission_id, measure_id, submeasure_id): son = self.request_son externals = son["externals"] with model.session_scope() as session: user_session = self.get_user_session(session) response = ( session.query(model.Response) .get((submission_id, measure_id))) if response is None: raise errors.MissingDocError("No such response") org = response.submission.organisation policy = user_session.policy.derive({ 'org': org, 'surveygroups': org.surveygroups, }) policy.verify('surveygroup_interact') policy.verify('attachment_add') for external in externals: url = external.get('url', '').strip() file_name = external.get('file_name', '').strip() if url == '' and file_name == '': continue if url == '': raise errors.ModelError( "URL required for link '%s'" % file_name) if measure_id!=submeasure_id: attachment = model.Attachment( organisation=response.submission.organisation, response=response, url=url, file_name=file_name, submeasure_id=submeasure_id, storage='external' ) else: attachment = model.Attachment( organisation=response.submission.organisation, response=response, url=url, file_name=file_name, storage='external' ) session.add(attachment) self.get(submission_id, measure_id, submeasure_id)
def post(self, file_type): with model.session_scope() as session: user_session = self.get_user_session(session) policy = user_session.policy.derive({}) policy.verify('custom_query_preview') text = to_basestring(self.request.body) if not text: raise errors.ModelError("Query is empty") custom_query = model.CustomQuery(description="Preview", text=text) conf = self.get_config(session) yield self.export(custom_query, conf, file_type) self.finish()
def get_version(self, session, response, version): if not version: return None try: version = int(version) except ValueError: raise errors.ModelError("Invalid version number") if version == response.version: return None history = (session.query(model.ResponseHistory).get( (response.submission_id, response.measure_id, version))) if history is None: raise errors.MissingDocError("No such version") return history
def _update(self, user, son, session): ''' Apply user-provided data to the saved model. ''' update = updater(user, error_factory=errors.ModelError) update('email', son) update('email_interval', son) update('name', son) update('role', son) update('password', son) org = ( session.query(model.Organisation) .get(son.organisation.id)) if not org: raise errors.ModelError("No such organisation") user.organisation = org
def get_version(self, session, custom_query, version): if not version: return None try: version = int(version) except ValueError: raise errors.ModelError("Invalid version number") if version == custom_query.version: return None history = (session.query(model.CustomQueryHistory).get( (custom_query.id, version))) if history is None: raise errors.MissingDocError("No such version") return history
def check_approval(self, session, submission, approval): approval_set = self.approval_set(approval) n_relevant_responses = (session.query(model.Response).filter( model.Response.submission_id == submission.id, model.Response.approval.in_(approval_set)).count()) n_measures = (session.query(model.QnodeMeasure.measure_id).join( model.QuestionNode).filter( model.QuestionNode.survey_id == submission.survey_id, model.QuestionNode.program_id == submission.program_id, model.QnodeMeasure.program_id == submission.program_id, model.QnodeMeasure.qnode_id == model.QuestionNode.id, model.QuestionNode.deleted == False).distinct().count()) if n_relevant_responses < n_measures: raise errors.ModelError( "%d of %d responses are incomplete" % (n_measures - n_relevant_responses, n_measures))
def post(self): ''' Check the strength of a password. ''' if 'password' not in self.request_son: raise errors.ModelError("Please specify a password") strength, threshold, improvements = test_password( self.request_son['password']) son = { 'threshold': threshold, 'strength': strength, 'improvements': improvements } self.set_header("Content-Type", "application/json") self.write(json_encode(son)) self.finish()