def fetch_lock(task_id): """Fetch the time (in seconds) until the current user's lock on a task expires. """ if not current_user.is_authenticated: return abort(401) task = task_repo.get_task(task_id) if not task: return abort(400) scheduler, timeout = get_project_scheduler_and_timeout(task.project_id) ttl = None if scheduler in (Schedulers.locked, Schedulers.user_pref): task_locked_by_user = has_lock(task.id, current_user.id, timeout) if task_locked_by_user: locks = get_locks(task.id, timeout) ttl = locks.get(str(current_user.id)) if not ttl: return abort(404) seconds_to_expire = float(ttl) - time() res = json.dumps({'success': True, 'expires': seconds_to_expire}) return Response(res, 200, mimetype='application/json')
def task_gold(project_id=None): """Make task gold""" if not current_user.is_authenticated: return abort(401) project = project_repo.get(project_id) if project is None or not (current_user.admin or current_user.id in project.owners_ids): raise Forbidden task_data = request.json task_id = task_data['task_id'] task = task_repo.get_task(task_id) if task.project_id != project_id: raise Forbidden task.calibration = 1 task.exported = True task.state = 'ongoing' preprocess_task_run(project_id, task_id, task_data) info = task_data['info'] if data_access_levels: task.gold_answers = upload_gold_data(task, project_id, info, task_id) else: task.gold_answers = info task_repo.update(task) return Response(json.dumps({'success': True}), 200, mimetype="application/json")
def post(self): """Add User ID to task as a favorite.""" try: self.valid_args() data = json.loads(request.data) if (len(data.keys()) != 1) or ('task_id' not in data.keys()): raise AttributeError if current_user.is_anonymous(): raise Unauthorized uid = current_user.id tasks = task_repo.get_task_favorited(uid, data['task_id']) if len(tasks) == 1: task = tasks[0] if len(tasks) == 0: task = task_repo.get_task(data['task_id']) if task is None: raise NotFound if task.fav_user_ids is None: task.fav_user_ids = [uid] else: task.fav_user_ids.append(uid) task_repo.update(task) self._log_changes(None, task) return Response(json.dumps(task.dictize()), 200, mimetype='application/json') except Exception as e: return error.format_exception( e, target=self.__class__.__name__.lower(), action='POST')
def _update_object(self, taskrun): """Update task_run object with user id or ip.""" # validate the task and project for that taskrun are ok task = task_repo.get_task(taskrun.task_id) if task is None: # pragma: no cover raise Forbidden('Invalid task_id') if (task.project_id != taskrun.project_id): raise Forbidden('Invalid project_id') if _check_task_requested_by_user(taskrun, sentinel.master) is False: raise Forbidden('You must request a task first!') # Add the user info so it cannot post again the same taskrun if current_user.is_anonymous(): taskrun.user_ip = request.remote_addr else: taskrun.user_id = current_user.id if request.headers.get('remote_mobile_addr') is not None : taskrun.user_ip = request.headers.get('remote_mobile_addr') if current_user.is_anonymous() else None else: if request.headers.getlist("X-Forwarded-For"): user_ip = request.headers.getlist("X-Forwarded-For")[0] taskrun.user_ip = user_ip print "user_ip %s." % user_ip if ',' in user_ip: ips = user_ip.split(",") for ip in ips: taskrun.user_ip = ip print "taskrun.user_ip %s." % taskrun.user_ip
def _file_upload(self, data): """Method that must be overriden by the class to allow file uploads for only a few classes.""" cls_name = self.__class__.__name__.lower() content_type_text = 'application/json' request_headers = request.headers.get('Content-Type') if request_headers is None: request_headers = [] if ((content_type_text in request_headers) and cls_name in self.allowed_classes_upload): data = dict() enc = json.loads(request.data) data['id'] = enc['task_id'] task = task_repo.get_task(enc['task_id']) data['project_id'] = task.project_id task.state = "completed" task.flagged = 1 task_repo.update(task) data['state'] = 'completed' data['flagged'] = 1 data['project_id'] = 1 flag_task = FlaggedTask(project_id=data['project_id'], task_id=data['id'], user_id=current_user.id, reason=enc['reason']) flagged_task_repo.save(flag_task) return data else: return None
def acquire_reserve_task_lock(project_id, task_id, user_id, timeout, pipeline=None, execute=True): task = task_repo.get_task(task_id) project = project_repo.get(project_id) if not (task and project and project.info.get("sched", "default") in [Schedulers.task_queue]): return False reserve_task_config = project.info.get("reserve_tasks", {}).get("category", []) category_exist = reserve_task_config and all( task.info.get(field, False) for field in reserve_task_config) if not category_exist: return False category = [ "{}:{}".format(field, task.info.get(field)) for field in reserve_task_config ] category = ":".join(category) redis_conn = sentinel.master pipeline = pipeline or redis_conn.pipeline(transaction=True) lock_manager = LockManager(redis_conn, timeout) if lock_manager.acquire_reserve_task_lock(project_id, task_id, user_id, category): current_app.logger.info( "Acquired reserve task lock. project %s, task %s, user %s, category %s", project_id, task_id, user_id, category) return True return False
def get_service_request(task_id, service_name, major_version, minor_version): """Proxy service call""" proxy_service_config = current_app.config.get('PROXY_SERVICE_CONFIG', None) task = task_repo.get_task(task_id) project = project_repo.get(task.project_id) if not (task and proxy_service_config and service_name and major_version and minor_version): return abort(400) timeout = project.info.get('timeout', ContributionsGuard.STAMP_TTL) task_locked_by_user = has_lock(task.id, current_user.id, timeout) payload = request.json if isinstance(request.json, dict) else None if payload and task_locked_by_user: service = _get_valid_service(task_id, service_name, payload, proxy_service_config) if isinstance(service, dict): url = '{}/{}/{}/{}'.format(proxy_service_config['uri'], service_name, major_version, minor_version) headers = service.get('headers') ret = requests.post(url, headers=headers, json=payload['data']) return Response(ret.content, 200, mimetype="application/json") current_app.logger.info( 'Task id {} with lock-status {} by user {} with this payload {} failed.' .format(task_id, task_locked_by_user, current_user.id, payload)) return abort(403)
def _after_save(self, original_data, instance): mark_if_complete(instance.task_id, instance.project_id) task = task_repo.get_task(instance.task_id) gold_answers = get_gold_answers(task) update_gold_stats(instance.user_id, instance.task_id, original_data, gold_answers) update_quiz(instance.project_id, original_data['info'], gold_answers)
def update_task_state(conn, task_id): task = task_repo.get_task(id=task_id) if task and task.calibration == 1: set_task_export(task_id) return sql_query = ("UPDATE task SET state=\'completed\' \ where id=%s") % task_id conn.execute(sql_query)
def _update_object(self, taskrun): """Update task_run object with user id or ip.""" task = task_repo.get_task(taskrun.task_id) guard = ContributionsGuard(sentinel.master) self._validate_project_and_task(taskrun, task) self._ensure_task_was_requested(task, guard) self._add_user_info(taskrun) self._add_created_timestamp(taskrun, task, guard)
def select_contributable_task(project, user_id, **kwargs): sched, _ = get_scheduler_and_timeout(project) with_user_pref = sched == Schedulers.user_pref sql = locked_task_sql(project.id, user_id, **kwargs) rows = session.execute( sql, dict(project_id=project.id, user_id=user_id, limit=1)) for row in rows: return task_repo.get_task(row.id) return {}
def mark_if_complete(task_id, project_id): project = project_repo.get(project_id) task = task_repo.get_task(task_id) # gold tasks never complete if task and task.calibration == 1: return if is_task_completed(task_id): update_task_state(task_id)
def _update_object(self, taskrun): """Update task_run object with user id or ip.""" task = task_repo.get_task(taskrun.task_id) guard = ContributionsGuard(sentinel.master) self._validate_project_and_task(taskrun, task) self._ensure_task_was_requested(task, guard) self._add_user_info(taskrun) self._add_timestamps(taskrun, task, guard)
def update_gold_stats(user_id, task_id, data): task = task_repo.get_task(task_id) # TODO: read gold_answer from s3 if task.calibration: answer_fields = get_project_data(task.project_id)['info'].get( 'answer_fields', {}) answer = data['info'] _update_gold_stats(task.project_id, user_id, answer_fields, task.gold_answers, answer)
def _update_object(self, taskrun): """Update task_run object with user id or ip.""" self.check_can_post(taskrun.project_id, taskrun.task_id, get_user_id_or_ip()) task = task_repo.get_task(taskrun.task_id) guard = ContributionsGuard(sentinel.master) self._validate_project_and_task(taskrun, task) self._ensure_task_was_requested(task, guard) self._add_user_info(taskrun) self._add_created_timestamp(taskrun, task, guard)
def create(taskrun=None): project = project_repo.get(task_repo.get_task(taskrun.task_id).app_id) if (current_user.is_anonymous() and project.allow_anonymous_contributors is False): return False authorized = task_repo.count_task_runs_with(app_id=taskrun.app_id, task_id=taskrun.task_id, user_id=taskrun.user_id, user_ip=taskrun.user_ip) <= 0 if not authorized: raise abort(403) return authorized
def update_gold_stats(user_id, task_id, data, gold_answers=None): task = task_repo.get_task(task_id) if not task.calibration: return if gold_answers is None: gold_answers = get_gold_answers(task) answer_fields = get_project_data(task.project_id)['info'].get( 'answer_fields', {}) answer = data['info'] _update_gold_stats(task.project_id, user_id, answer_fields, gold_answers, answer)
def update_quiz(task_id, project_id, answer): project = project_repo.get(project_id) user = user_repo.get(current_user.id) if not user.get_quiz_in_progress(project): return task = task_repo.get_task(task_id) if task.gold_answers == answer: user.add_quiz_right_answer(project) else: user.add_quiz_wrong_answer(project) user_repo.update(user)
def release_user_locks_for_project(user_id, project_id): user_tasks = get_user_tasks(user_id, TIMEOUT) user_task_ids = user_tasks.keys() results = get_task_ids_project_id(user_task_ids) task_ids = [] for task_id, task_project_id in zip(user_task_ids, results): if not task_project_id: task_project_id = task_repo.get_task(task_id).project_id if int(task_project_id) == project_id: release_lock(task_id, user_id, TIMEOUT) task_ids.append(task_id) current_app.logger.info('released user id {} locks on tasks {}'.format(user_id, task_ids)) return task_ids
def encrypt_task_response_data(task_id, project_id, data): content = None task = task_repo.get_task(task_id) if not (task and isinstance(task.info, dict) and 'private_json__encrypted_payload' in task.info): return content project = get_project_data(project_id) secret = get_encryption_key(project) cipher = AESWithGCM(secret) content = json.dumps(data) content = cipher.encrypt(content.encode('utf8')).decode('utf8') return content
def _update_object(self, taskrun): """Update task_run object with user id or ip.""" # validate the task and app for that taskrun are ok task = task_repo.get_task(taskrun.task_id) if task is None: # pragma: no cover raise Forbidden('Invalid task_id') if (task.app_id != taskrun.app_id): raise Forbidden('Invalid app_id') # Add the user info so it cannot post again the same taskrun if current_user.is_anonymous(): taskrun.user_ip = request.remote_addr else: taskrun.user_id = current_user.id
def select_contributable_task(project, user_id, **kwargs): sched, _ = get_scheduler_and_timeout(project) with_user_pref = sched == Schedulers.user_pref kwargs['filter_user_prefs'] = with_user_pref params = dict(project_id=project.id, user_id=user_id, limit=1) if with_user_pref: params['assign_user'] = None sql = locked_task_sql(project.id, user_id, **kwargs) rows = session.execute(sql, params) for row in rows: return task_repo.get_task(row.id) return {}
def encrypted_task_payload(project_id, task_id): """Proxy to decrypt encrypted task payload""" current_app.logger.info( 'Project id {}, task id {}, decrypt task payload.'.format( project_id, task_id)) signature = request.args.get('task-signature') if not signature: current_app.logger.exception( 'Project id {}, task id {} has no signature.'.format( project_id, task_id)) raise Forbidden('No signature') size_signature = len(signature) if size_signature > TASK_SIGNATURE_MAX_SIZE: current_app.logger.exception( 'Project id {}, task id {} invalid task signature. Signature length {} exceeds max allowed length {}.' \ .format(project_id, task_id, size_signature, TASK_SIGNATURE_MAX_SIZE)) raise Forbidden('Invalid signature') project = get_project_data(project_id) if not project: current_app.logger.exception('Invalid project id {}.'.format( project_id, task_id)) raise BadRequest('Invalid Project') timeout = project['info'].get('timeout', ContributionsGuard.STAMP_TTL) payload = signer.loads(signature, max_age=timeout) task_id = payload.get('task_id', 0) validate_task(project, task_id, current_user.id) ## decrypt encrypted task data under private_json__encrypted_payload try: secret = get_encryption_key(project) task = task_repo.get_task(task_id) content = task.info.get('private_json__encrypted_payload') if content: cipher = AESWithGCM(secret) content = cipher.decrypt(content) else: content = '' except Exception as e: current_app.logger.exception( 'Project id {} task {} decrypt encrypted data {}'.format( project_id, task_id, e)) raise InternalServerError('An Error Occurred') response = Response(content, content_type='application/json') return response
def _update_object(self, taskrun): """Update task_run object with user id or ip.""" task = task_repo.get_task(taskrun.task_id) # validate the task and project for that taskrun are ok if task is None: # pragma: no cover raise Forbidden('Invalid task_id') if task.project_id != taskrun.project_id: raise Forbidden('Invalid project_id') if _check_task_requested_by_user(taskrun, sentinel.master) is False: raise Forbidden('You must request a task first!') # validate and modify taskrun attributes self._add_user_info(taskrun) self._add_timestamps(taskrun, task, sentinel.master)
def _update_object(self, inst): ensure_authorized_to('create', Result) if not inst.task_id: raise BadRequest('Invalid task id') task_id = inst.task_id results = result_repo.get_by(task_id=task_id) if results: raise BadRequest('Record is already present') task = task_repo.get_task(task_id) if not task or task.state != 'completed': raise BadRequest('Invalid task') inst.created = make_timestamp() inst.project_id = task.project_id inst.task_run_ids = [tr.id for tr in task.task_runs] inst.last_version = True
def _update_object(self, taskrun): """Update task_run object with user id or ip.""" # validate the task and project for that taskrun are ok task = task_repo.get_task(taskrun.task_id) if task is None: # pragma: no cover raise Forbidden('Invalid task_id') if (task.project_id != taskrun.project_id): raise Forbidden('Invalid project_id') if _check_task_requested_by_user(taskrun, sentinel.master) is False: raise Forbidden('You must request a task first!') # Add the user info so it cannot post again the same taskrun if current_user.is_anonymous(): taskrun.user_ip = request.remote_addr else: taskrun.user_id = current_user.id
def _update_object(self, taskrun): """Update task_run object with user id or ip.""" # validate the task and app for that taskrun are ok task = task_repo.get_task(taskrun.task_id) if task is None: # pragma: no cover raise Forbidden('Invalid task_id') if (task.app_id != taskrun.app_id): raise Forbidden('Invalid app_id') if _check_task_requested_by_user(taskrun, sentinel.master) is False: raise Forbidden('You must request a task first!') # Add the user info so it cannot post again the same taskrun if current_user.is_anonymous(): taskrun.user_ip = request.remote_addr else: taskrun.user_id = current_user.id
def test_get_returns_result_after_increasig_redundancy(self): """Test get method returns a result if after increasing redundancy""" n_answers = 1 task = TaskFactory.create(n_answers=n_answers) task_run = TaskRunFactory.create(task=task) result = self.result_repo.filter_by(project_id=1) err_msg = "There should be a result" assert len(result) == 1, err_msg result = result[0] assert result.project_id == 1, err_msg assert result.task_id == task.id, err_msg assert len(result.task_run_ids) == n_answers, err_msg err_msg = "The task_run id is missing in the results array" for tr_id in result.task_run_ids: assert tr_id == task_run.id, err_msg # Increase redundancy tmp = task_repo.get_task(task.id) tmp.n_answers = 2 task_repo.update(task) err_msg = "There should be only one result" results = result_repo.filter_by(project_id=1) assert len(results) == 1, err_msg task_run_2 = TaskRunFactory.create(task=task) err_msg = "There should be 1 results" results = result_repo.filter_by(project_id=1) assert len(results) == 1, err_msg err_msg = "There should be 2 results" results = result_repo.filter_by(project_id=1, last_version=False) assert len(results) == 2, err_msg assert results[1].project_id == 1, err_msg assert results[1].task_id == task.id, err_msg err_msg = "First result should have only one task run ID" assert len(results[0].task_run_ids) == 1, err_msg err_msg = "Second result should have only two task run IDs" assert len(results[1].task_run_ids) == 2, err_msg err_msg = "The task_run id is missing in the results array" for tr_id in results[1].task_run_ids: assert tr_id in [task_run.id, task_run_2.id], err_msg
def on_taskrun_submit(mapper, conn, target): """Update the task.state when n_answers condition is met.""" # Get project details sql_query = ( 'select name, short_name, published, webhook, info, category_id \ from project where id=%s') % target.project_id results = conn.execute(sql_query) tmp = dict() for r in results: tmp['name'] = r.name tmp['short_name'] = r.short_name _published = r.published tmp['info'] = r.info _webhook = r.webhook tmp['category_id'] = r.category_id tmp['id'] = target.project_id project_public = dict() project_public.update(Project().to_public_json(tmp)) project_public['action_updated'] = 'TaskCompleted' sched.after_save(target, conn) add_user_contributed_to_feed(conn, target.user_id, project_public) # golden tasks never complete; bypass update to task.state # mark task as exported false for each task run submissions task = task_repo.get_task(id=target.task_id) if task.calibration: if task.exported and _published: sql_query = ("""UPDATE task SET exported=False \ WHERE id=%s;""") % (task.id) conn.execute(sql_query) return is_completed = is_task_completed(conn, target.task_id, target.project_id) if is_completed: update_task_state(conn, target.task_id) check_and_send_task_notifications(target.project_id, conn) if is_completed and _published: update_feed(project_public) result_id = create_result(conn, target.project_id, target.task_id) project_private = dict() project_private.update(project_public) project_private['webhook'] = _webhook push_webhook(project_private, target.task_id, result_id)
def validate_task(project, task_id, user_id): """Confirm task payload is valid and user is authorized to access task.""" task = task_repo.get_task(task_id) if not task or task.project_id != project['id']: raise BadRequest('Task does not exist') if current_user.admin: return True if has_lock(task_id, user_id, project['info'].get('timeout', ContributionsGuard.STAMP_TTL)): return True if user_id in project['owners_ids']: return True raise Forbidden('FORBIDDEN')
def get_task_id_and_duration_for_project_user(project_id, user_id): user_tasks = get_user_tasks(user_id, TIMEOUT) user_task_ids = user_tasks.keys() results = get_task_ids_project_id(user_task_ids) max_seconds_task_id = -1 max_seconds_remaining = float('-inf') for task_id, task_project_id in zip(user_task_ids, results): if not task_project_id: task_project_id = task_repo.get_task(task_id).project_id save_task_id_project_id(task_id, task_project_id, 2 * TIMEOUT) if int(task_project_id) == project_id: seconds_remaining = LockManager.seconds_remaining( user_tasks[task_id]) if seconds_remaining > max_seconds_remaining: max_seconds_task_id = int(task_id) max_seconds_remaining = seconds_remaining if max_seconds_task_id > 0: return max_seconds_task_id, max_seconds_remaining return None, -1
def check_allowed(user_id, task_id, project, file_url): task = task_repo.get_task(task_id) if not task or task.project_id != project['id']: raise BadRequest('Task does not exist') if file_url not in task.info.values(): raise Forbidden('Invalid task content') if current_user.admin: return True if has_lock(task_id, user_id, project['info'].get('timeout', ContributionsGuard.STAMP_TTL)): return True if user_id in project['owners_ids']: return True raise Forbidden('FORBIDDEN')
def test_get_last_version(self): """Test API result returns always latest version.""" result = self.create_result() project = project_repo.get(result.project_id) task = task_repo.get_task(result.task_id) task.n_answers = 2 TaskRunFactory.create(task=task, project=project) result = result_repo.get_by(project_id=project.id) assert result.last_version is True, result.last_version result_id = result.id results = result_repo.filter_by(project_id=project.id, last_version=False) assert len(results) == 2, len(results) for r in results: if r.id == result_id: assert r.last_version is True, r.last_version else: assert r.last_version is False, r.last_version
def task_gold(project_id=None): """Make task gold""" try: if not current_user.is_authenticated: return abort(401) project = project_repo.get(project_id) # Allow project owner, sub-admin co-owners, and admins to update Gold tasks. is_gold_access = (current_user.subadmin and current_user.id in project.owners_ids) or current_user.admin if project is None or not is_gold_access: raise Forbidden if request.method == 'POST': task_data = json.loads( request.form['request_json'] ) if 'request_json' in request.form else request.json task_id = task_data['task_id'] task = task_repo.get_task(task_id) if task.project_id != project_id: raise Forbidden preprocess_task_run(project_id, task_id, task_data) info = task_data['info'] set_gold_answers(task, info) task_repo.update(task) response_body = json.dumps({'success': True}) else: task = sched.select_task_for_gold_mode(project, current_user.id) if task: task = task.dictize() sign_task(task) response_body = json.dumps(task) return Response(response_body, 200, mimetype="application/json") except Exception as e: return error.format_exception(e, target='taskgold', action=request.method)