def before_add_task_event(mapper, conn, target):
redis_conn = sentinel.master
if cached_projects.get_project_scheduler(target.project_id) == Schedulers.user_pref:
if not redis_conn.hget('updated_project_ids', target.project_id):
redis_conn.hset('updated_project_ids', target.project_id, make_timestamp())
else:
if cached_projects.overall_progress(target.project_id) == 100:
redis_conn.hset('updated_project_ids', target.project_id, make_timestamp())
def create_result(conn, project_id, task_id):
"""Create a result for the given project and task."""
sql_query = ("SELECT id FROM task_run WHERE project_id=%s \
AND task_id=%s") % (project_id, task_id)
results = conn.execute(sql_query)
task_run_ids = ", ".join(str(tr.id) for tr in results)
sql_query = ("""SELECT id FROM result WHERE project_id=%s \
AND task_id=%s;""") % (project_id, task_id)
results = conn.execute(sql_query)
for r in results:
if r:
# Update result
sql_query = ("""UPDATE result SET last_version=false \
WHERE id=%s;""") % (r.id)
conn.execute(sql_query)
sql_query = """INSERT INTO result
(created, project_id, task_id, task_run_ids, last_version)
VALUES ('%s', %s, %s, '{%s}', %s);""" % (make_timestamp(),
project_id,
task_id,
task_run_ids,
True)
conn.execute(sql_query)
sql_query = """SELECT id FROM result \
WHERE project_id=%s AND task_id=%s""" % (project_id, task_id)
results = conn.execute(sql_query)
for r in results:
return r.id
def _sign_in_user(user, next_url=None):
brand = current_app.config['BRAND']
if not user:
flash(
gettext('There was a problem signing you in. '
'Please contact your {} administrator.'.format(brand)),
'error')
return redirect(url_for('home.home'))
if not user.enabled:
flash(
gettext('Your account is disabled. '
'Please contact your {} administrator.'.format(brand)),
'error')
return redirect(url_for('home.home'))
login_user(user, remember=False)
user.last_login = model.make_timestamp()
user_repo.update(user)
next_url = (next_url or is_own_url_or_else(request.args.get('next'),
url_for('home.home'))
or url_for('home.home'))
if (current_app.config.get('MAILCHIMP_API_KEY')
and newsletter.ask_user_to_subscribe(user)):
return redirect_content_type(
url_for('account.newsletter_subscribe', next=next_url))
return redirect_content_type(next_url)
def extend_task_presented_timestamp_expiry(self, task, user):
"""Extend expiry time for task presented time for user."""
key = self._create_presented_time_key(task, user)
task_presented = self.conn.get(key)
if task_presented is not None:
return self.conn.expire(key, self.STAMP_TTL)
else:
return self.conn.setEx(key, self.STAMP_TTL, make_timestamp())
def _sign_in_user(user):
login_user(user, remember=False)
user.last_login = model.make_timestamp()
user_repo.update(user)
if newsletter.ask_user_to_subscribe(user):
return redirect_content_type(url_for('account.newsletter_subscribe',
next=request.args.get('next')))
return redirect_content_type(request.args.get("next") or
url_for("home.home"))
def _sign_in_user(user):
login_user(user, remember=False)
user.last_login = model.make_timestamp()
user_repo.update(user)
next_url = (is_own_url_or_else(request.args.get('next'),
url_for('home.home'))
or url_for('home.home'))
if (current_app.config.get('MAILCHIMP_API_KEY')
and newsletter.ask_user_to_subscribe(user)):
return redirect_content_type(
url_for('account.newsletter_subscribe', next=next_url))
return redirect_content_type(next_url)
def _update_object(self, inst):
ensure_authorized_to('create', Result)
if not inst.task_id:
raise BadRequest('Invalid task id')
task_id = inst.task_id
results = result_repo.get_by(task_id=task_id)
if results:
raise BadRequest('Record is already present')
task = task_repo.get_task(task_id)
if not task or task.state != 'completed':
raise BadRequest('Invalid task')
inst.created = make_timestamp()
inst.project_id = task.project_id
inst.task_run_ids = [tr.id for tr in task.task_runs]
inst.last_version = True
def update_task_state(self, project_id, n_answers):
# Create temp tables for completed tasks
sql = text('''
CREATE TEMP TABLE complete_tasks ON COMMIT DROP AS (
SELECT task.id, array_agg(task_run.id) as task_runs
FROM task, task_run
WHERE task_run.task_id=task.id
AND task.project_id=:project_id
GROUP BY task.id
having COUNT(task_run.id) >=:n_answers);
''')
self.db.session.execute(sql, dict(n_answers=n_answers,
project_id=project_id))
# Set state to completed
sql = text('''
UPDATE task SET state='completed'
FROM complete_tasks
WHERE complete_tasks.id=task.id;
''')
self.db.session.execute(sql)
# Deactivate previous tasks' results (if available)
# (redundancy was decreased)
sql = text('''UPDATE result set last_version=false
WHERE task_id IN (SELECT id FROM complete_tasks);''')
self.db.session.execute(sql)
# Insert result rows (last_version=true)
sql = text('''
INSERT INTO result
(created, project_id, task_id, task_run_ids, last_version) (
SELECT :ts, :project_id, complete_tasks.id,
complete_tasks.task_runs, true
FROM complete_tasks);''')
self.db.session.execute(sql, dict(project_id=project_id,
ts=make_timestamp()))
# Create temp table for incomplete tasks
sql = text('''
CREATE TEMP TABLE incomplete_tasks ON COMMIT DROP AS (
SELECT task.id
FROM task
WHERE task.project_id=:project_id
AND task.id not IN (SELECT id FROM complete_tasks));
''')
self.db.session.execute(sql, dict(project_id=project_id))
# Delete results for incomplete tasks (Redundancy Increased)
sql = text('''DELETE FROM result
WHERE result.task_id IN (SELECT id FROM incomplete_tasks);
''')
self.db.session.execute(sql)
def grant_access_with_api_key(secure_app):
from pybossa.core import user_repo
import pybossa.model as model
from flask import _request_ctx_stack
apikey = None
if not secure_app:
apikey = request.args.get('api_key', None)
if 'Authorization' in request.headers:
apikey = request.headers.get('Authorization')
if apikey:
user = user_repo.get_by(api_key=apikey)
if user and user.enabled:
user.last_login = model.make_timestamp()
user_repo.update(user)
_request_ctx_stack.top.user = user
def clean_project(project_id, skip_tasks=False):
"""Remove everything from a project."""
from pybossa.core import task_repo
from pybossa.model import make_timestamp
n_tasks = 0
if not skip_tasks:
print "Deleting tasks"
sql = 'delete from task where project_id=%s' % project_id
db.engine.execute(sql)
else:
sql = 'select count(id) as n from task where project_id=%s' % project_id
result = db.engine.execute(sql)
for row in result:
n_tasks = row.n
sql = 'delete from task_run where project_id=%s' % project_id
db.engine.execute(sql)
sql = 'delete from result where project_id=%s' % project_id
db.engine.execute(sql)
sql = 'delete from counter where project_id=%s' % project_id
db.engine.execute(sql)
sql = 'delete from project_stats where project_id=%s' % project_id
db.engine.execute(sql)
sql = """INSERT INTO project_stats
(project_id, n_tasks, n_task_runs, n_results, n_volunteers,
n_completed_tasks, overall_progress, average_time,
n_blogposts, last_activity, info)
VALUES (%s, %s, 0, 0, 0, 0, 0, 0, 0, 0, '{}');""" % (project_id,
n_tasks)
db.engine.execute(sql)
if skip_tasks:
tasks = task_repo.filter_tasks_by(project_id=project_id, limit=100)
last_id = tasks[len(tasks) - 1].id
while (len(tasks) > 0):
for task in tasks:
sql = (
"insert into counter(created, project_id, task_id, n_task_runs) \
VALUES (TIMESTAMP '%s', %s, %s, 0)" %
(make_timestamp(), project_id, task.id))
db.engine.execute(sql)
tasks = task_repo.filter_tasks_by(project_id=project_id,
limit=100,
last_id=last_id)
if (len(tasks) > 0):
last_id = tasks[len(tasks) - 1].id
print "Project has been cleaned"
def clean_project(project_id, skip_tasks=False):
"""Remove everything from a project."""
from pybossa.core import task_repo
from pybossa.model import make_timestamp
n_tasks = 0
if not skip_tasks:
print "Deleting tasks"
sql = 'delete from task where project_id=%s' % project_id
db.engine.execute(sql)
else:
sql = 'select count(id) as n from task where project_id=%s' % project_id
result = db.engine.execute(sql)
for row in result:
n_tasks = row.n
sql = 'delete from task_run where project_id=%s' % project_id
db.engine.execute(sql)
sql = 'delete from result where project_id=%s' % project_id
db.engine.execute(sql)
sql = 'delete from counter where project_id=%s' % project_id
db.engine.execute(sql)
sql = 'delete from project_stats where project_id=%s' % project_id
db.engine.execute(sql)
sql = """INSERT INTO project_stats
(project_id, n_tasks, n_task_runs, n_results, n_volunteers,
n_completed_tasks, overall_progress, average_time,
n_blogposts, last_activity, info)
VALUES (%s, %s, 0, 0, 0, 0, 0, 0, 0, 0, '{}');""" % (project_id,
n_tasks)
db.engine.execute(sql)
if skip_tasks:
tasks = task_repo.filter_tasks_by(project_id=project_id, limit=100)
last_id = tasks[len(tasks)-1].id
while(len(tasks) > 0):
for task in tasks:
sql= ("insert into counter(created, project_id, task_id, n_task_runs) \
VALUES (TIMESTAMP '%s', %s, %s, 0)"
% (make_timestamp(), project_id, task.id))
db.engine.execute(sql)
tasks = task_repo.filter_tasks_by(project_id=project_id,
limit=100,
last_id=last_id)
if (len(tasks) > 0):
last_id = tasks[len(tasks)-1].id
print "Project has been cleaned"
def enable_user(user_id=None):
"""Set enabled flag to True for user_id."""
if user_id:
user = user_repo.get(user_id)
if user:
# avoid enabling admin/subadmin user by subadmin with direct url
if not can_update_user_info(current_user, user)[0]:
return abort(403)
user.enabled = True
user.last_login = make_timestamp()
user_repo.update(user)
msg = generate_manage_user_email(user, "enable")
if msg:
mail_queue.enqueue(send_mail, msg)
return redirect(url_for(".manageusers"))
msg = "User not found"
return format_error(msg, 404)
def update_task_state(self, project_id):
# Create temp tables for completed tasks
sql = text('''
CREATE TEMP TABLE complete_tasks ON COMMIT DROP AS (
SELECT task.id, array_agg(task_run.id) as task_runs
FROM task, task_run
WHERE task_run.task_id=task.id
AND task.project_id=:project_id
AND task.calibration!=1
GROUP BY task.id
having COUNT(task_run.id) >= task.n_answers);
''')
self.db.session.execute(sql, dict(project_id=project_id))
# Set state to completed
sql = text('''
UPDATE task SET state='completed'
FROM complete_tasks
WHERE complete_tasks.id=task.id;
''')
self.db.session.execute(sql)
sql = text('''
INSERT INTO result
(created, project_id, task_id, task_run_ids, last_version) (
SELECT :ts, :project_id, completed_no_results.id,
completed_no_results.task_runs, true
FROM ( SELECT task.id as id,
array_agg(task_run.id) as task_runs
FROM task, task_run
WHERE task.state = 'completed'
AND task_run.task_id = task.id
AND NOT EXISTS (SELECT 1 FROM result
WHERE result.task_id = task.id)
AND task.project_id=:project_id
GROUP BY task.id
) as completed_no_results
);''')
self.db.session.execute(sql, dict(project_id=project_id,
ts=make_timestamp()))
def payload(self, project, task):
return dict(fired_at=make_timestamp(),
project_short_name=project.short_name,
project_id=project.id,
event='task_completed',
task_id=task.id)
def stamp(self, task, user):
key = self._create_key(task, user)
self.conn.setex(key, self.STAMP_TTL, make_timestamp())
def stamp(self, task, user):
"""Cache the time that a task was requested by a client
for a given user.
"""
key = self._create_key(task, user)
self.conn.setex(key, self.STAMP_TTL, make_timestamp())
def decrease_task_counter(mapper, conn, target):
sql_query = (
"insert into counter(created, project_id, task_id, n_task_runs) \
VALUES (TIMESTAMP '%s', %s, %s, -1)" %
(make_timestamp(), target.project_id, target.task_id))
conn.execute(sql_query)
def payload(self, project, task):
return dict(fired_at=make_timestamp(),
project_short_name=project.short_name,
project_id= project.id,
event='task_completed',
task_id=task.id)
def stamp(self, task, user):
key = self._create_key(task, user)
self.conn.setex(key, self.STAMP_TTL, make_timestamp())
def stamp_presented_time(self, task, user):
"""Cache the time that a task was presented on a client."""
key = self._create_presented_time_key(task, user)
self.conn.setex(key, self.STAMP_TTL, make_timestamp())
def decrease_task_counter(mapper, conn, target):
sql_query = ("insert into counter(created, project_id, task_id, n_task_runs) \
VALUES (TIMESTAMP '%s', %s, %s, -1)"
% (make_timestamp(), target.project_id, target.task_id))
conn.execute(sql_query)
def update_audit_data(mapper, conn, target):
try:
if not isinstance(target.info, dict):
return
data = target.info
if not data.has_key('result'):
return
sql_query = (
'select id from project where category_id in (select category_id \
from project where id=%s) and id != %s') % (target.project_id,
target.project_id)
print(sql_query)
result = conn.execute(sql_query)
print(3432432)
one = result.fetchone()
if len(one) <= 0:
return
project_id = one[0]
print(one)
info = {}
print(target.__dict__)
info['project_id'] = project_id
info['task_id'] = target.task_id
info['user_id'] = target.user_id
info['user_ip'] = ""
info['finish_time'] = target.finish_time
info['result'] = data['result']
info['answers'] = data['answers']
info['question'] = data['question']
info['link'] = data['link']
info['url_m'] = data['url_m']
info['url_b'] = data['url_b']
sql_query = (
"insert into task(created, project_id, state, quorum, calibration, \
priority_0, info, n_answers) values (TIMESTAMP '%s', %s, 'ongoing', 0, 0, 0, '%s', 30) RETURNING id;"
% (make_timestamp(), project_id, json.dumps(info)))
print(sql_query)
result = conn.execute(sql_query)
id_of_new_row = result.fetchone()[0]
print(id_of_new_row)
sql_query = (
"insert into counter(created, project_id, task_id, n_task_runs) \
VALUES (TIMESTAMP '%s', %s, %s, 0)" %
(make_timestamp(), project_id, id_of_new_row))
print(sql_query)
conn.execute(sql_query)
print(sql_query)
"""Update PYBOSSA feed with new task."""
sql_query = ('select name, short_name, info from project \
where id=%s') % project_id
results = conn.execute(sql_query)
obj = dict(action_updated='Task')
tmp = dict()
for r in results:
tmp['id'] = project_id
tmp['name'] = r.name
tmp['short_name'] = r.short_name
tmp['info'] = r.info
tmp = Project().to_public_json(tmp)
obj.update(tmp)
update_feed(obj)
except:
raise
