def warm_app(id, short_name, featured=False):
if id not in apps_cached:
cached_apps.get_app(short_name)
cached_apps.n_tasks(id)
n_task_runs = cached_apps.n_task_runs(id)
cached_apps.overall_progress(id)
cached_apps.last_activity(id)
cached_apps.n_completed_tasks(id)
cached_apps.n_volunteers(id)
if n_task_runs >= 1000 or featured:
print "Getting stats for %s as it has %s task runs" % (short_name, n_task_runs)
stats.get_stats(id, app.config.get('GEO'))
apps_cached.append(id)
def warm_app(id, short_name, featured=False):
if id not in apps_cached:
cached_apps.get_app(short_name)
cached_apps.n_tasks(id)
n_task_runs = cached_apps.n_task_runs(id)
cached_apps.overall_progress(id)
cached_apps.last_activity(id)
cached_apps.n_completed_tasks(id)
cached_apps.n_volunteers(id)
if n_task_runs >= 1000 or featured:
print("Getting stats for %s as it has %s task runs" % (short_name, n_task_runs))
stats.get_stats(id)
apps_cached.append(id)
def get_project_stats(_id, short_name): # pragma: no cover
"""Get stats for project."""
import pybossa.cache.projects as cached_projects
import pybossa.cache.project_stats as stats
from flask import current_app
cached_projects.get_project(short_name)
cached_projects.n_tasks(_id)
cached_projects.n_task_runs(_id)
cached_projects.overall_progress(_id)
cached_projects.last_activity(_id)
cached_projects.n_completed_tasks(_id)
cached_projects.n_volunteers(_id)
stats.get_stats(_id, current_app.config.get('GEO'))
def get_project_stats(_id, short_name): # pragma: no cover
"""Get stats for project."""
import pybossa.cache.projects as cached_projects
import pybossa.cache.project_stats as stats
from flask import current_app
cached_projects.get_project(short_name)
cached_projects.n_tasks(_id)
cached_projects.n_task_runs(_id)
cached_projects.overall_progress(_id)
cached_projects.last_activity(_id)
cached_projects.n_completed_tasks(_id)
cached_projects.n_volunteers(_id)
stats.get_stats(_id, current_app.config.get('GEO'))
def warm_project(_id, short_name, featured=False):
if _id not in projects_cached:
cached_projects.get_project(short_name)
cached_projects.n_tasks(_id)
n_task_runs = cached_projects.n_task_runs(_id)
cached_projects.overall_progress(_id)
cached_projects.last_activity(_id)
cached_projects.n_completed_tasks(_id)
cached_projects.n_volunteers(_id)
if n_task_runs >= 1000 or featured:
# print ("Getting stats for %s as it has %s task runs" %
# (short_name, n_task_runs))
stats.get_stats(_id, app.config.get('GEO'))
projects_cached.append(_id)
def warm_project(_id, short_name, featured=False):
if _id not in projects_cached:
cached_projects.get_project(short_name)
cached_projects.n_tasks(_id)
n_task_runs = cached_projects.n_task_runs(_id)
cached_projects.overall_progress(_id)
cached_projects.last_activity(_id)
cached_projects.n_completed_tasks(_id)
cached_projects.n_volunteers(_id)
if n_task_runs >= 1000 or featured:
# print ("Getting stats for %s as it has %s task runs" %
# (short_name, n_task_runs))
stats.get_stats(_id, app.config.get('GEO'))
projects_cached.append(_id)
def update_stats(project_id, period='2 week'):
"""Update the stats of a given project."""
hours, hours_anon, hours_auth, max_hours, \
max_hours_anon, max_hours_auth = stats_hours(project_id, period)
users, anon_users, auth_users = stats_users(project_id, period)
dates, dates_anon, dates_auth = stats_dates(project_id, period)
sum(dates.values())
sorted(dates.iteritems(), key=operator.itemgetter(0))
dates_stats = stats_format_dates(project_id, dates,
dates_anon, dates_auth)
hours_stats = stats_format_hours(project_id, hours, hours_anon, hours_auth,
max_hours, max_hours_anon, max_hours_auth)
users_stats = stats_format_users(project_id, users, anon_users, auth_users)
data = dict(dates_stats=dates_stats,
hours_stats=hours_stats,
users_stats=users_stats)
ps = session.query(ProjectStats).filter_by(project_id=project_id).first()
n_tasks = cached_projects.n_tasks(project_id)
n_task_runs = cached_projects.n_task_runs(project_id)
n_results = cached_projects.n_results(project_id)
overall_progress = cached_projects.overall_progress(project_id)
last_activity = cached_projects.last_activity(project_id)
n_volunteers = cached_projects.n_volunteers(project_id)
n_completed_tasks = cached_projects.n_completed_tasks(project_id)
average_time = cached_projects.average_contribution_time(project_id)
n_blogposts = cached_projects.n_blogposts(project_id)
if ps is None:
ps = ProjectStats(project_id=project_id, info=data,
n_tasks=n_tasks,
n_task_runs=n_task_runs,
n_results=n_results,
n_volunteers=n_volunteers,
n_completed_tasks=n_completed_tasks,
average_time=average_time,
overall_progress=overall_progress,
n_blogposts=n_blogposts,
last_activity=last_activity)
db.session.add(ps)
else:
ps.info = data
ps.n_tasks = n_tasks
ps.n_task_runs = n_task_runs
ps.overall_progress = overall_progress
ps.last_activity = last_activity
ps.n_results = n_results
ps.n_completed_tasks = n_completed_tasks
ps.n_volunteers = n_volunteers
ps.average_time = average_time
ps.n_blogposts = n_blogposts
db.session.commit()
return dates_stats, hours_stats, users_stats
def draft_projects(user_id):
"""Return draft projects for user_id."""
sql = text('''
SELECT project.id, project.name, project.short_name, project.description,
owner_id,
project.info
FROM project
WHERE project.owner_id=:user_id
AND (project.info->>'task_presenter') IS NULL
GROUP BY project.id, project.name, project.short_name,
project.description;
''')
projects_draft = []
results = session.execute(sql, dict(user_id=user_id))
for row in results:
project = dict(id=row.id,
name=row.name,
short_name=row.short_name,
owner_id=row.owner_id,
description=row.description,
overall_progress=overall_progress(row.id),
n_tasks=n_tasks(row.id),
n_volunteers=n_volunteers(row.id),
info=row.info)
projects_draft.append(project)
return projects_draft
def hidden_projects(user_id):
"""Return hidden projects for user_id."""
sql = text(
"""
SELECT project.id, project.name, project.short_name, project.description,
project.owner_id,
project.info
FROM project, task
WHERE project.id=task.project_id AND project.owner_id=:user_id AND
project.hidden=1 AND (project.info->>'task_presenter') IS NOT NULL
GROUP BY project.id, project.name, project.short_name,
project.description;"""
)
projects_published = []
results = session.execute(sql, dict(user_id=user_id))
for row in results:
project = dict(
id=row.id,
name=row.name,
short_name=row.short_name,
owner_id=row.owner_id,
description=row.description,
overall_progress=overall_progress(row.id),
n_tasks=n_tasks(row.id),
n_volunteers=n_volunteers(row.id),
info=row.info,
)
projects_published.append(project)
return projects_published
def update_stats(project_id, period='2 week'):
"""Update the stats of a given project."""
hours, hours_anon, hours_auth, max_hours, \
max_hours_anon, max_hours_auth = stats_hours(project_id, period)
users, anon_users, auth_users = stats_users(project_id, period)
dates, dates_anon, dates_auth = stats_dates(project_id, period)
sum(dates.values())
sorted(iter(dates.items()), key=operator.itemgetter(0))
dates_stats = stats_format_dates(project_id, dates,
dates_anon, dates_auth)
hours_stats = stats_format_hours(project_id, hours, hours_anon, hours_auth,
max_hours, max_hours_anon, max_hours_auth)
users_stats = stats_format_users(project_id, users, anon_users, auth_users)
data = dict(dates_stats=dates_stats,
hours_stats=hours_stats,
users_stats=users_stats)
ps = session.query(ProjectStats).filter_by(project_id=project_id).first()
n_tasks = cached_projects.n_tasks(project_id)
n_task_runs = cached_projects.n_task_runs(project_id)
n_results = cached_projects.n_results(project_id)
overall_progress = cached_projects.overall_progress(project_id)
last_activity = cached_projects.last_activity(project_id)
n_volunteers = cached_projects.n_volunteers(project_id)
n_completed_tasks = cached_projects.n_completed_tasks(project_id)
average_time = cached_projects.average_contribution_time(project_id)
n_blogposts = cached_projects.n_blogposts(project_id)
if ps is None:
ps = ProjectStats(project_id=project_id, info=data,
n_tasks=n_tasks,
n_task_runs=n_task_runs,
n_results=n_results,
n_volunteers=n_volunteers,
n_completed_tasks=n_completed_tasks,
average_time=average_time,
overall_progress=overall_progress,
n_blogposts=n_blogposts,
last_activity=last_activity)
db.session.add(ps)
else:
ps.info = data
ps.n_tasks = n_tasks
ps.n_task_runs = n_task_runs
ps.overall_progress = overall_progress
ps.last_activity = last_activity
ps.n_results = n_results
ps.n_completed_tasks = n_completed_tasks
ps.n_volunteers = n_volunteers
ps.average_time = average_time
ps.n_blogposts = n_blogposts
db.session.commit()
return dates_stats, hours_stats, users_stats
def hidden_projects(user_id):
"""Return hidden projects for user_id."""
sql = text('''
SELECT project.id, project.name, project.short_name, project.description,
project.owner_id,
project.info
FROM project, task
WHERE project.id=task.project_id AND project.owner_id=:user_id AND
project.hidden=1 AND project.info LIKE('%task_presenter%')
GROUP BY project.id, project.name, project.short_name,
project.description,
project.info;''')
projects_published = []
results = session.execute(sql, dict(user_id=user_id))
for row in results:
project = dict(id=row.id,
name=row.name,
short_name=row.short_name,
owner_id=row.owner_id,
description=row.description,
overall_progress=overall_progress(row.id),
n_tasks=n_tasks(row.id),
n_volunteers=n_volunteers(row.id),
info=json.loads(row.info))
projects_published.append(project)
return projects_published
def projects_contributed(user_id):
"""Return projects that user_id has contributed to."""
sql = text(
"""
WITH apps_contributed as
(SELECT DISTINCT(project_id) FROM task_run
WHERE user_id=:user_id)
SELECT project.id, project.name, project.short_name, project.owner_id,
project.description, project.info FROM project, apps_contributed
WHERE project.id=apps_contributed.project_id ORDER BY project.name DESC;
"""
)
results = session.execute(sql, dict(user_id=user_id))
projects_contributed = []
for row in results:
project = dict(
id=row.id,
name=row.name,
short_name=row.short_name,
owner_id=row.owner_id,
description=row.description,
overall_progress=overall_progress(row.id),
n_tasks=n_tasks(row.id),
n_volunteers=n_volunteers(row.id),
info=row.info,
)
projects_contributed.append(project)
return projects_contributed
def published_projects(user_id, args=None):
"""Return published projects for user_id."""
if args is None:
args = dict(column=None, order=None)
sort_args = dict(column=args.get("column"), order=args.get("order"))
if sort_args.get("order") not in ("asc", "desc"):
sort_args["order"] = "desc"
sort_args["column"] = allowed_project_columns.get(sort_args["column"], "created")
sql = text('''
SELECT project.id, project.name, project.short_name, project.description,
project.owner_id,
project.owners_ids,
project.info
FROM project
WHERE project.published=true
AND :user_id = ANY (project.owners_ids::int[])
order by {column} {order};
'''.format(**sort_args))
projects_published = []
results = session.execute(sql, dict(user_id=user_id))
for row in results:
project = dict(id=row.id, name=row.name, short_name=row.short_name,
owner_id=row.owner_id,
owners_ids=row.owners_ids,
description=row.description,
overall_progress=overall_progress(row.id),
n_tasks=n_tasks(row.id),
n_volunteers=n_volunteers(row.id),
info=row.info)
projects_published.append(project)
return projects_published
def draft_projects(user_id):
"""Return draft projects for user_id."""
sql = text('''
SELECT project.id, project.name, project.short_name, project.description,
project.owner_id,
project.owners_ids,
project.info
FROM project
WHERE project.published=false
AND :user_id = ANY (project.owners_ids::int[]);
''')
projects_draft = []
results = session.execute(sql, dict(user_id=user_id))
for row in results:
project = dict(id=row.id,
name=row.name,
short_name=row.short_name,
owner_id=row.owner_id,
owners_ids=row.owners_ids,
description=row.description,
overall_progress=overall_progress(row.id),
n_tasks=n_tasks(row.id),
n_volunteers=n_volunteers(row.id),
info=row.info)
projects_draft.append(project)
return projects_draft
def projects_contributed(user_id, order_by='name'):
"""Return projects that user_id has contributed to."""
sql = text('''
WITH projects_contributed as
(SELECT project_id, MAX(finish_time) as last_contribution FROM task_run
WHERE user_id=:user_id GROUP BY project_id)
SELECT project.id, project.name as name, project.short_name, project.owner_id,
project.description, project.info, project.owners_ids
FROM project, projects_contributed
WHERE project.id=projects_contributed.project_id ORDER BY {} DESC;
'''.format(order_by))
results = session.execute(sql, dict(user_id=user_id))
projects_contributed = []
for row in results:
project = dict(id=row.id,
name=row.name,
short_name=row.short_name,
owner_id=row.owner_id,
owners_ids=row.owners_ids,
description=row.description,
overall_progress=overall_progress(row.id),
n_tasks=n_tasks(row.id),
n_volunteers=n_volunteers(row.id),
info=row.info)
projects_contributed.append(project)
return projects_contributed
def test_n_volunteers(self):
"""Test CACHE PROJECTS n_volunteers returns the sum of the anonymous
plus registered volunteers that contributed to a project"""
project = self.create_project_with_contributors(anonymous=2, registered=3, two_tasks=True)
total_volunteers = cached_projects.n_volunteers(project.id)
err_msg = "Volunteers is %s, it should be 5" % total_volunteers
assert total_volunteers == 5, err_msg
def test_n_volunteers(self):
"""Test CACHE PROJECTS n_volunteers returns the sum of the anonymous
plus registered volunteers that contributed to a project"""
project = self.create_project_with_contributors(anonymous=2, registered=3, two_tasks=True)
total_volunteers = cached_projects.n_volunteers(project.id)
err_msg = "Volunteers is %s, it should be 5" % total_volunteers
assert total_volunteers == 5, err_msg
def draft_projects(user_id):
"""Return draft projects for user_id."""
sql = text('''
SELECT *
FROM project
WHERE project.published=false
AND :user_id = ANY (project.owners_ids::int[]);
''')
projects_draft = []
results = session.execute(sql, dict(user_id=user_id))
for row in results:
project = dict(row)
project['n_tasks'] = n_tasks(row.id)
project['n_volunteers'] = n_volunteers(row.id)
project['overall_progress'] = overall_progress(row.id)
projects_draft.append(project)
return projects_draft
def draft_projects(user_id):
"""Return draft projects for user_id."""
sql = text('''
SELECT *
FROM project
WHERE project.published=false
AND :user_id = ANY (project.owners_ids::int[]);
''')
projects_draft = []
results = session.execute(sql, dict(user_id=user_id))
for row in results:
project = dict(row)
project['n_tasks'] = n_tasks(row.id)
project['n_volunteers'] = n_volunteers(row.id)
project['overall_progress'] = overall_progress(row.id)
projects_draft.append(project)
return projects_draft
def flush_task_runs(project_short_name, confirmed):
project = cached_projects.get_project(project_short_name)
if current_user.admin or project.owner_id == current_user.id:
if confirmed == "confirmed":
associated_task_runs = TaskRun.query.filter_by(project_id=project.id).all()
for task_run in associated_task_runs:
db.session.delete(task_run)
pass
db.session.commit()
# Iterate over all tasks associated with the project, and mark them as 'ongoing'
# Some tasks might be marked as 'completed' if enough task_runs were done
associated_tasks = Task.query.filter_by(project_id=project.id).all()
for task in associated_tasks:
if task.state != u"ongoing":
task.state = u"ongoing"
db.session.commit()
# Reset project data in the cache
cached_projects.clean_project(project.id)
# Note: The cache will hold the old data about the users who contributed
# to the tasks associated with this projects till the User Cache Timeout.
# Querying the list of contributors to this project, and then individually updating
# their cache after that will be a very expensive query, hence we will avoid that
# for the time being.
flash('All Task Runs associated with this project have been successfully deleted.', 'success')
return redirect(url_for('project.task_settings', short_name = project_short_name))
elif confirmed == "unconfirmed":
# Obtain data required by the project profile renderer
(project, owner, n_tasks, n_task_runs,
overall_progress, last_activity, n_results) = projects_view.project_by_shortname(project_short_name)
return render_template('geotagx/projects/delete_task_run_confirmation.html',
project=project,
owner=owner,
n_tasks=n_tasks,
n_task_runs=n_task_runs,
overall_progress=overall_progress,
last_activity=last_activity,
n_results=n_results,
n_completed_tasks=cached_projects.n_completed_tasks(project.id),
n_volunteers=cached_projects.n_volunteers(project.id))
else:
abort(404)
else:
abort(404)
def projects_contributed(user_id):
"""Return projects that user_id has contributed to."""
sql = text('''
WITH projects_contributed as
(SELECT DISTINCT(project_id) FROM task_run
WHERE user_id=:user_id)
SELECT * FROM project, projects_contributed
WHERE project.id=projects_contributed.project_id ORDER BY project.name DESC;
''')
results = session.execute(sql, dict(user_id=user_id))
projects_contributed = []
for row in results:
project = dict(row)
project['n_tasks'] = n_tasks(row.id)
project['n_volunteers'] = n_volunteers(row.id)
project['overall_progress'] = overall_progress(row.id),
projects_contributed.append(project)
return projects_contributed
def projects_contributed(user_id, order_by='name'):
"""Return projects that user_id has contributed to."""
sql = text('''
WITH projects_contributed as
(SELECT project_id, MAX(finish_time) as last_contribution FROM task_run
WHERE user_id=:user_id GROUP BY project_id)
SELECT * FROM project, projects_contributed
WHERE project.id=projects_contributed.project_id ORDER BY {} DESC;
'''.format(order_by))
results = session.execute(sql, dict(user_id=user_id))
projects_contributed = []
for row in results:
project = dict(row)
project['n_tasks'] = n_tasks(row.id)
project['n_volunteers'] = n_volunteers(row.id)
project['overall_progress'] = overall_progress(row.id)
projects_contributed.append(project)
return projects_contributed
def projects_contributed(user_id, order_by='name'):
"""Return projects that user_id has contributed to."""
sql = text('''
WITH projects_contributed as
(SELECT project_id, MAX(finish_time) as last_contribution FROM task_run
WHERE user_id=:user_id GROUP BY project_id)
SELECT * FROM project, projects_contributed
WHERE project.id=projects_contributed.project_id ORDER BY {} DESC;
'''.format(order_by))
results = session.execute(sql, dict(user_id=user_id))
projects_contributed = []
for row in results:
project = dict(row)
project['n_tasks'] = n_tasks(row.id)
project['n_volunteers'] = n_volunteers(row.id)
project['overall_progress'] = overall_progress(row.id)
projects_contributed.append(project)
return projects_contributed
def draft_projects(user_id):
"""Return draft projects for user_id."""
sql = text('''
SELECT project.id, project.name, project.short_name, project.description,
project.owner_id,
project.info
FROM project
WHERE project.owner_id=:user_id
AND project.published=false;
''')
projects_draft = []
results = session.execute(sql, dict(user_id=user_id))
for row in results:
project = dict(id=row.id, name=row.name, short_name=row.short_name,
owner_id=row.owner_id,
description=row.description,
overall_progress=overall_progress(row.id),
n_tasks=n_tasks(row.id),
n_volunteers=n_volunteers(row.id),
info=row.info)
projects_draft.append(project)
return projects_draft
def published_projects(user_id):
"""Return published projects for user_id."""
sql = text('''
SELECT project.id, project.name, project.short_name, project.description,
project.owner_id,
project.info
FROM project
WHERE project.published=true
AND project.owner_id=:user_id;
''')
projects_published = []
results = session.execute(sql, dict(user_id=user_id))
for row in results:
project = dict(id=row.id, name=row.name, short_name=row.short_name,
owner_id=row.owner_id,
description=row.description,
overall_progress=overall_progress(row.id),
n_tasks=n_tasks(row.id),
n_volunteers=n_volunteers(row.id),
info=row.info)
projects_published.append(project)
return projects_published
def projects_contributed(user_id):
"""Return projects that user_id has contributed to."""
sql = text('''
WITH projects_contributed as
(SELECT DISTINCT(project_id) FROM task_run
WHERE user_id=:user_id)
SELECT project.id, project.name, project.short_name, project.owner_id,
project.description, project.info FROM project, projects_contributed
WHERE project.id=projects_contributed.project_id ORDER BY project.name DESC;
''')
results = session.execute(sql, dict(user_id=user_id))
projects_contributed = []
for row in results:
project = dict(id=row.id, name=row.name, short_name=row.short_name,
owner_id=row.owner_id,
description=row.description,
overall_progress=overall_progress(row.id),
n_tasks=n_tasks(row.id),
n_volunteers=n_volunteers(row.id),
info=row.info)
projects_contributed.append(project)
return projects_contributed
def visualize(short_name, task_id):
"""Return a file with all the TaskRuns for a given Task"""
# Check if it a supported geotagx project whose schema we know
if 'GEOTAGX_SUPPORTED_PROJECTS_SCHEMA' in current_app.config.keys() \
and short_name in current_app.config['GEOTAGX_SUPPORTED_PROJECTS_SCHEMA'].keys():
# Check if the project exists
(project, owner, n_tasks, n_task_runs,
overall_progress, last_activity) = projects_view.project_by_shortname(short_name)
ensure_authorized_to('read', project)
redirect_to_password = projects_view._check_if_redirect_to_password(project)
if redirect_to_password:
return redirect_to_password
# Check if the task belongs to the project and exists
task = task_repo.get_task_by(project_id=project.id, id=task_id)
if task:
taskruns = task_repo.filter_task_runs_by(task_id=task_id, project_id=project.id)
results = [tr.dictize() for tr in taskruns]
return render_template('geotagx/projects/task_runs_visualize.html',
project=project,
owner=owner,
n_tasks=n_tasks,
n_task_runs=n_task_runs,
overall_progress=overall_progress,
last_activity=last_activity,
n_completed_tasks=cached_projects.n_completed_tasks(project.id),
n_volunteers=cached_projects.n_volunteers(project.id),
task_info = task.info,
task_runs_json = results,
geotagx_project_template_schema = \
current_app.config['GEOTAGX_SUPPORTED_PROJECTS_SCHEMA'][short_name])
else:
return abort(404)
else:
return abort(404)
def published_projects(user_id):
"""Return published projects for user_id."""
sql = text('''
SELECT project.id, project.name, project.short_name, project.description,
project.owner_id,
project.info
FROM project, task
WHERE project.id=task.project_id AND project.owner_id=:user_id AND
project.hidden=0 AND project.info LIKE('%task_presenter%')
GROUP BY project.id, project.name, project.short_name,
project.description,
project.info;''')
projects_published = []
results = session.execute(sql, dict(user_id=user_id))
for row in results:
project = dict(id=row.id, name=row.name, short_name=row.short_name,
owner_id=row.owner_id,
description=row.description,
overall_progress=overall_progress(row.id),
n_tasks=n_tasks(row.id),
n_volunteers=n_volunteers(row.id),
info=json.loads(row.info))
projects_published.append(project)
return projects_published