def warm_app(id, short_name, featured=False):
if id not in apps_cached:
cached_apps.get_app(short_name)
cached_apps.n_tasks(id)
n_task_runs = cached_apps.n_task_runs(id)
cached_apps.overall_progress(id)
cached_apps.last_activity(id)
cached_apps.n_completed_tasks(id)
cached_apps.n_volunteers(id)
if n_task_runs >= 1000 or featured:
print("Getting stats for %s as it has %s task runs" % (short_name, n_task_runs))
stats.get_stats(id)
apps_cached.append(id)
def warm_app(id, short_name, featured=False):
if id not in apps_cached:
cached_apps.get_app(short_name)
cached_apps.n_tasks(id)
n_task_runs = cached_apps.n_task_runs(id)
cached_apps.overall_progress(id)
cached_apps.last_activity(id)
cached_apps.n_completed_tasks(id)
cached_apps.n_volunteers(id)
if n_task_runs >= 1000 or featured:
print "Getting stats for %s as it has %s task runs" % (short_name, n_task_runs)
stats.get_stats(id, app.config.get('GEO'))
apps_cached.append(id)
def warm_project(_id, short_name, featured=False):
if _id not in projects_cached:
cached_projects.get_project(short_name)
cached_projects.n_tasks(_id)
n_task_runs = cached_projects.n_task_runs(_id)
cached_projects.overall_progress(_id)
cached_projects.last_activity(_id)
cached_projects.n_completed_tasks(_id)
cached_projects.n_volunteers(_id)
if n_task_runs >= 1000 or featured:
# print ("Getting stats for %s as it has %s task runs" %
# (short_name, n_task_runs))
stats.get_stats(_id, app.config.get('GEO'))
projects_cached.append(_id)
def get_project_stats(_id, short_name): # pragma: no cover
"""Get stats for project."""
import pybossa.cache.projects as cached_projects
import pybossa.cache.project_stats as stats
from flask import current_app
cached_projects.get_project(short_name)
cached_projects.n_tasks(_id)
cached_projects.n_task_runs(_id)
cached_projects.overall_progress(_id)
cached_projects.last_activity(_id)
cached_projects.n_completed_tasks(_id)
cached_projects.n_volunteers(_id)
stats.get_stats(_id, current_app.config.get('GEO'))
def warm_project(_id, short_name, featured=False):
if _id not in projects_cached:
cached_projects.get_project(short_name)
cached_projects.n_tasks(_id)
n_task_runs = cached_projects.n_task_runs(_id)
cached_projects.overall_progress(_id)
cached_projects.last_activity(_id)
cached_projects.n_completed_tasks(_id)
cached_projects.n_volunteers(_id)
if n_task_runs >= 1000 or featured:
# print ("Getting stats for %s as it has %s task runs" %
# (short_name, n_task_runs))
stats.get_stats(_id, app.config.get('GEO'))
projects_cached.append(_id)
def get_project_stats(_id, short_name): # pragma: no cover
"""Get stats for project."""
import pybossa.cache.projects as cached_projects
import pybossa.cache.project_stats as stats
from flask import current_app
cached_projects.get_project(short_name)
cached_projects.n_tasks(_id)
cached_projects.n_task_runs(_id)
cached_projects.overall_progress(_id)
cached_projects.last_activity(_id)
cached_projects.n_completed_tasks(_id)
cached_projects.n_volunteers(_id)
stats.get_stats(_id, current_app.config.get('GEO'))
def send_weekly_stats_project(project_id):
from pybossa.cache.project_stats import update_stats, get_stats
from pybossa.core import project_repo
from datetime import datetime
project = project_repo.get(project_id)
if project.owner.subscribed is False:
return "Owner does not want updates by email"
update_stats(project_id)
dates_stats, hours_stats, users_stats = get_stats(project_id,
geo=True,
period='1 week')
subject = "Weekly Update: %s" % project.name
timeout = current_app.config.get('TIMEOUT')
# Max number of completed tasks
n_completed_tasks = 0
xy = zip(*dates_stats[3]['values'])
n_completed_tasks = max(xy[1])
# Most active day
xy = zip(*dates_stats[0]['values'])
active_day = [xy[0][xy[1].index(max(xy[1]))], max(xy[1])]
active_day[0] = datetime.fromtimestamp(active_day[0]/1000).strftime('%A')
body = render_template('/account/email/weeklystats.md',
project=project,
dates_stats=dates_stats,
hours_stats=hours_stats,
users_stats=users_stats,
n_completed_tasks=n_completed_tasks,
active_day=active_day,
config=current_app.config)
html = render_template('/account/email/weeklystats.html',
project=project,
dates_stats=dates_stats,
hours_stats=hours_stats,
users_stats=users_stats,
active_day=active_day,
n_completed_tasks=n_completed_tasks,
config=current_app.config)
mail_dict = dict(recipients=[project.owner.email_addr],
subject=subject,
body=body,
html=html)
job = dict(name=send_mail,
args=[mail_dict],
kwargs={},
timeout=timeout,
queue='high')
enqueue_job(job)
def send_weekly_stats_project(project_id):
from pybossa.cache.project_stats import update_stats, get_stats
from pybossa.core import project_repo
from datetime import datetime
project = project_repo.get(project_id)
if project.owner.subscribed is False:
return "Owner does not want updates by email"
update_stats(project_id)
dates_stats, hours_stats, users_stats = get_stats(project_id,
geo=True,
period='1 week')
subject = "Weekly Update: %s" % project.name
timeout = current_app.config.get('TIMEOUT')
# Max number of completed tasks
n_completed_tasks = 0
xy = zip(*dates_stats[3]['values'])
n_completed_tasks = max(xy[1])
# Most active day
xy = zip(*dates_stats[0]['values'])
active_day = [xy[0][xy[1].index(max(xy[1]))], max(xy[1])]
active_day[0] = datetime.fromtimestamp(active_day[0] / 1000).strftime('%A')
body = render_template('/account/email/weeklystats.md',
project=project,
dates_stats=dates_stats,
hours_stats=hours_stats,
users_stats=users_stats,
n_completed_tasks=n_completed_tasks,
active_day=active_day,
config=current_app.config)
html = render_template('/account/email/weeklystats.html',
project=project,
dates_stats=dates_stats,
hours_stats=hours_stats,
users_stats=users_stats,
active_day=active_day,
n_completed_tasks=n_completed_tasks,
config=current_app.config)
mail_dict = dict(recipients=[project.owner.email_addr],
subject=subject,
body=body,
html=html)
job = dict(name=send_mail,
args=[mail_dict],
kwargs={},
timeout=timeout,
queue='high')
enqueue_job(job)
def progress(short_name):
"""Return progress for each volume and template."""
category = project_repo.get_category_by(short_name=short_name)
if not category: # pragma: no cover
abort(404)
tmpl_index = {t['id']: t for t in category.info.get('templates', [])}
vol_index = {v['id']: v for v in category.info.get('volumes', [])}
data = {v_id: {t_id: None for t_id in tmpl_index} for v_id in vol_index}
projects = project_repo.filter_by(category_id=category.id)
for project in projects:
ps = project_stats.get_stats(project.id, full=True)
try:
row = data[project.info['volume_id']]
row[project.info['template_id']] = ps.overall_progress
except KeyError:
continue
# Replace IDs with names and flatten
flat_data = []
for vol_id in data:
row = {'Volume': vol_index[vol_id]['name']}
for tmpl_id in data[vol_id]:
tmpl_name = tmpl_index[tmpl_id]['name']
if tmpl_name == 'Volume':
tmpl_name = '_Volume'
row[tmpl_name] = data[vol_id][tmpl_id]
flat_data.append(row)
if request.args.get('csv'):
df = pandas.DataFrame(flat_data)
df.set_index('Volume', inplace=True)
csv = df.to_csv(encoding='utf8')
response = dict(progress=csv)
return handle_content_type(response)
response = dict(progress=flat_data)
return handle_content_type(response)
def test_03_stats(self):
"""Test STATS stats method works"""
self.prepare_data()
today = str(datetime.date.today())
hour = int(datetime.datetime.utcnow().strftime('%H'))
date_ms = time.mktime(time.strptime(today, "%Y-%m-%d")) * 1000
anon = 0
auth = 0
tr1 = TaskRunFactory.create(task=self.project.tasks[0])
tr2 = TaskRunFactory.create(task=self.project.tasks[1])
user = user_repo.get(tr1.user_id)
user.restrict = True
user_repo.update(user)
stats.update_stats(self.project.id)
dates_stats, hours_stats, user_stats = stats.get_stats(self.project.id)
for item in dates_stats:
if item['label'] == 'Anon + Auth':
assert item['values'][-1][0] == date_ms, item['values'][0][0]
assert item['values'][-1][
1] == 10, "There should be 10 answers"
if item['label'] == 'Anonymous':
assert item['values'][-1][0] == date_ms, item['values'][0][0]
anon = item['values'][-1][1]
if item['label'] == 'Authenticated':
assert item['values'][-1][0] == date_ms, item['values'][0][0]
auth = item['values'][-1][1]
if item['label'] == 'Total Tasks':
assert item['values'][-1][0] == date_ms, item['values'][0][0]
assert item['values'][-1][1] == 4, "There should be 4 tasks"
if item['label'] == 'Expected Answers':
assert item['values'][0][0] == date_ms, item['values'][0][0]
for i in item['values']:
assert i[1] == 100, "Each date should have 100 answers"
assert item['values'][0][
1] == 100, "There should be 10 answers"
assert auth + anon == 10, "date stats sum of auth and anon should be 10"
max_hours = 0
for item in hours_stats:
if item['label'] == 'Anon + Auth':
max_hours = item['max']
print(item)
assert item['max'] == 10, item['max']
assert item['max'] == 10, "Max hours value should be 10"
for i in item['values']:
if i[0] == hour:
assert i[1] == 10, "There should be 10 answers"
assert i[2] == 5, "The size of the bubble should be 5"
else:
assert i[1] == 0, "There should be 0 answers"
assert i[2] == 0, "The size of the buggle should be 0"
if item['label'] == 'Anonymous':
anon = item['max']
for i in item['values']:
if i[0] == hour:
assert i[1] == anon, "There should be anon answers"
assert i[2] == (
anon * 5
) / max_hours, "The size of the bubble should be 5"
else:
assert i[1] == 0, "There should be 0 answers"
assert i[2] == 0, "The size of the buggle should be 0"
if item['label'] == 'Authenticated':
auth = item['max']
for i in item['values']:
if i[0] == hour:
assert i[1] == auth, "There should be anon answers"
assert i[2] == (
auth * 5
) / max_hours, "The size of the bubble should be 5"
else:
assert i[1] == 0, "There should be 0 answers"
assert i[2] == 0, "The size of the buggle should be 0"
assert auth + anon == 10, "date stats sum of auth and anon should be 8"
err_msg = "user stats sum of auth and anon should be 7"
assert user_stats['n_anon'] + user_stats['n_auth'] == 7, err_msg
for u in user_stats['auth']['top5']:
assert u['restrict'] is False, u
def test_03_stats(self):
"""Test STATS stats method works"""
today = unicode(datetime.date.today())
hour = int(datetime.datetime.utcnow().strftime('%H'))
date_ms = time.mktime(time.strptime(today, "%Y-%m-%d")) * 1000
anon = 0
auth = 0
TaskRunFactory.create(task=self.project.tasks[0])
TaskRunFactory.create(task=self.project.tasks[1])
dates_stats, hours_stats, user_stats = stats.get_stats(self.project.id)
for item in dates_stats:
if item['label'] == 'Anon + Auth':
assert item['values'][0][0] == date_ms, item['values'][0][0]
assert item['values'][0][1] == 10, "There should be 10 answers"
if item['label'] == 'Anonymous':
assert item['values'][0][0] == date_ms, item['values'][0][0]
anon = item['values'][0][1]
if item['label'] == 'Authenticated':
assert item['values'][0][0] == date_ms, item['values'][0][0]
auth = item['values'][0][1]
if item['label'] == 'Total Tasks':
assert item['values'][0][0] == date_ms, item['values'][0][0]
assert item['values'][0][1] == 4, "There should be 4 tasks"
if item['label'] == 'Expected Answers':
assert item['values'][0][0] == date_ms, item['values'][0][0]
for i in item['values']:
assert i[1] == 100, "Each date should have 100 answers"
assert item['values'][0][1] == 100, "There should be 10 answers"
assert auth + anon == 10, "date stats sum of auth and anon should be 10"
max_hours = 0
for item in hours_stats:
if item['label'] == 'Anon + Auth':
max_hours = item['max']
print item
assert item['max'] == 10, item['max']
assert item['max'] == 10, "Max hours value should be 10"
for i in item['values']:
if i[0] == hour:
assert i[1] == 10, "There should be 10 answers"
assert i[2] == 5, "The size of the bubble should be 5"
else:
assert i[1] == 0, "There should be 0 answers"
assert i[2] == 0, "The size of the buggle should be 0"
if item['label'] == 'Anonymous':
anon = item['max']
for i in item['values']:
if i[0] == hour:
assert i[1] == anon, "There should be anon answers"
assert i[2] == (anon * 5) / max_hours, "The size of the bubble should be 5"
else:
assert i[1] == 0, "There should be 0 answers"
assert i[2] == 0, "The size of the buggle should be 0"
if item['label'] == 'Authenticated':
auth = item['max']
for i in item['values']:
if i[0] == hour:
assert i[1] == auth, "There should be anon answers"
assert i[2] == (auth * 5) / max_hours, "The size of the bubble should be 5"
else:
assert i[1] == 0, "There should be 0 answers"
assert i[2] == 0, "The size of the buggle should be 0"
assert auth + anon == 10, "date stats sum of auth and anon should be 8"
err_msg = "user stats sum of auth and anon should be 7"
assert user_stats['n_anon'] + user_stats['n_auth'] == 7, err_msg
def test_query_projectstats(self):
"""Test API query for project stats endpoint works"""
project_stats = []
projects = ProjectFactory.create_batch(3)
for project in projects:
for task in TaskFactory.create_batch(4, project=project, n_answers=3):
TaskRunFactory.create(task=task)
stats.update_stats(project.id)
ps = stats.get_stats(project.id, full=True)
project_stats.append(ps)
extra_stat_types = ['hours_stats', 'dates_stats', 'users_stats']
# As anon
url = '/api/projectstats'
res = self.app_get_json(url)
data = json.loads(res.data)
assert len(data) == 3, data
# Limits
res = self.app.get(url + "?limit=1")
data = json.loads(res.data)
assert len(data) == 1, data
# Keyset pagination
res = self.app.get(url + '?limit=1&last_id=' + str(projects[1].id))
data = json.loads(res.data)
assert len(data) == 1, len(data)
assert data[0]['id'] == project.id
# Errors
res = self.app.get(url + "?something")
err = json.loads(res.data)
err_msg = "AttributeError exception should be raised"
res.status_code == 415, err_msg
assert res.status_code == 415, err_msg
assert err['action'] == 'GET', err_msg
assert err['status'] == 'failed', err_msg
assert err['exception_cls'] == 'AttributeError', err_msg
# Desc filter
url = "/api/projectstats?orderby=wrongattribute"
res = self.app.get(url)
data = json.loads(res.data)
err_msg = "It should be 415."
assert data['status'] == 'failed', data
assert data['status_code'] == 415, data
assert 'has no attribute' in data['exception_msg'], data
# Order by
url = "/api/projectstats?orderby=id"
res = self.app.get(url)
data = json.loads(res.data)
err_msg = "It should get the last item first."
ps_by_id = sorted(project_stats, key=lambda x: x.id, reverse=False)
for i in range(len(project_stats)):
assert ps_by_id[i].id == data[i]['id']
# Desc filter
url = "/api/projectstats?orderby=id&desc=true"
res = self.app.get(url)
data = json.loads(res.data)
err_msg = "It should get the last item first."
ps_by_id = sorted(project_stats, key=lambda x: x.id, reverse=True)
for i in range(len(project_stats)):
assert ps_by_id[i].id == data[i]['id']
# Without full filter
url = "/api/projectstats"
res = self.app.get(url)
data = json.loads(res.data)
err_msg = "It should not return the full stats."
extra = [row['info'].get(_type) for _type in extra_stat_types
for row in data if row['info'].get(_type)]
assert not extra
# With full filter
url = "/api/projectstats?full=1"
res = self.app.get(url)
data = json.loads(res.data)
err_msg = "It should return full stats."
for i, row in enumerate(data):
for _type in extra_stat_types:
assert row['info'][_type] == project_stats[i].info[_type]
def prepare_stats(self):
project = ProjectFactory.create()
stats.update_stats(project.id)
return stats.get_stats(project.id, full=True)
def test_query_projectstats(self):
"""Test API query for project stats endpoint works"""
project_stats = []
projects = ProjectFactory.create_batch(3)
for project in projects:
for task in TaskFactory.create_batch(4,
project=project,
n_answers=3):
TaskRunFactory.create(task=task)
stats.update_stats(project.id)
ps = stats.get_stats(project.id, full=True)
project_stats.append(ps)
extra_stat_types = ['hours_stats', 'dates_stats', 'users_stats']
# As anon
url = '/api/projectstats'
res = self.app_get_json(url)
data = json.loads(res.data)
assert len(data) == 3, data
# Limits
res = self.app.get(url + "?limit=1")
data = json.loads(res.data)
assert len(data) == 1, data
# Keyset pagination
res = self.app.get(url + '?limit=1&last_id=' + str(projects[1].id))
data = json.loads(res.data)
assert len(data) == 1, len(data)
assert data[0]['id'] == project.id
# Errors
res = self.app.get(url + "?something")
err = json.loads(res.data)
err_msg = "AttributeError exception should be raised"
res.status_code == 415, err_msg
assert res.status_code == 415, err_msg
assert err['action'] == 'GET', err_msg
assert err['status'] == 'failed', err_msg
assert err['exception_cls'] == 'AttributeError', err_msg
# Desc filter
url = "/api/projectstats?orderby=wrongattribute"
res = self.app.get(url)
data = json.loads(res.data)
err_msg = "It should be 415."
assert data['status'] == 'failed', data
assert data['status_code'] == 415, data
assert 'has no attribute' in data['exception_msg'], data
# Order by
url = "/api/projectstats?orderby=id"
res = self.app.get(url)
data = json.loads(res.data)
err_msg = "It should get the last item first."
ps_by_id = sorted(project_stats, key=lambda x: x.id, reverse=False)
for i in range(len(project_stats)):
assert ps_by_id[i].id == data[i]['id']
# Desc filter
url = "/api/projectstats?orderby=id&desc=true"
res = self.app.get(url)
data = json.loads(res.data)
err_msg = "It should get the last item first."
ps_by_id = sorted(project_stats, key=lambda x: x.id, reverse=True)
for i in range(len(project_stats)):
assert ps_by_id[i].id == data[i]['id']
# Without full filter
url = "/api/projectstats"
res = self.app.get(url)
data = json.loads(res.data)
err_msg = "It should not return the full stats."
extra = [
row['info'].get(_type) for _type in extra_stat_types
for row in data if row['info'].get(_type)
]
assert not extra
# With full filter
url = "/api/projectstats?full=1"
res = self.app.get(url)
data = json.loads(res.data)
err_msg = "It should return full stats."
for i, row in enumerate(data):
for _type in extra_stat_types:
assert row['info'][_type] == project_stats[i].info[_type]
def prepare_stats(self, n=1):
project = ProjectFactory.create()
TaskFactory.create_batch(n, project=project)
stats.update_stats(project.id)
return stats.get_stats(project.id, full=True)
def prepare_stats(self, n=1): project = ProjectFactory.create() TaskFactory.create_batch(n, project=project) stats.update_stats(project.id) return stats.get_stats(project.id, full=True)
