def warm_app(id, short_name, featured=False): if id not in apps_cached: cached_apps.get_app(short_name) cached_apps.n_tasks(id) n_task_runs = cached_apps.n_task_runs(id) cached_apps.overall_progress(id) cached_apps.last_activity(id) cached_apps.n_completed_tasks(id) cached_apps.n_volunteers(id) if n_task_runs >= 1000 or featured: print("Getting stats for %s as it has %s task runs" % (short_name, n_task_runs)) stats.get_stats(id) apps_cached.append(id)
def warm_app(id, short_name, featured=False): if id not in apps_cached: cached_apps.get_app(short_name) cached_apps.n_tasks(id) n_task_runs = cached_apps.n_task_runs(id) cached_apps.overall_progress(id) cached_apps.last_activity(id) cached_apps.n_completed_tasks(id) cached_apps.n_volunteers(id) if n_task_runs >= 1000 or featured: print "Getting stats for %s as it has %s task runs" % (short_name, n_task_runs) stats.get_stats(id, app.config.get('GEO')) apps_cached.append(id)
def warm_project(_id, short_name, featured=False): if _id not in projects_cached: cached_projects.get_project(short_name) cached_projects.n_tasks(_id) n_task_runs = cached_projects.n_task_runs(_id) cached_projects.overall_progress(_id) cached_projects.last_activity(_id) cached_projects.n_completed_tasks(_id) cached_projects.n_volunteers(_id) if n_task_runs >= 1000 or featured: # print ("Getting stats for %s as it has %s task runs" % # (short_name, n_task_runs)) stats.get_stats(_id, app.config.get('GEO')) projects_cached.append(_id)
def get_project_stats(_id, short_name): # pragma: no cover """Get stats for project.""" import pybossa.cache.projects as cached_projects import pybossa.cache.project_stats as stats from flask import current_app cached_projects.get_project(short_name) cached_projects.n_tasks(_id) cached_projects.n_task_runs(_id) cached_projects.overall_progress(_id) cached_projects.last_activity(_id) cached_projects.n_completed_tasks(_id) cached_projects.n_volunteers(_id) stats.get_stats(_id, current_app.config.get('GEO'))
def send_weekly_stats_project(project_id): from pybossa.cache.project_stats import update_stats, get_stats from pybossa.core import project_repo from datetime import datetime project = project_repo.get(project_id) if project.owner.subscribed is False: return "Owner does not want updates by email" update_stats(project_id) dates_stats, hours_stats, users_stats = get_stats(project_id, geo=True, period='1 week') subject = "Weekly Update: %s" % project.name timeout = current_app.config.get('TIMEOUT') # Max number of completed tasks n_completed_tasks = 0 xy = zip(*dates_stats[3]['values']) n_completed_tasks = max(xy[1]) # Most active day xy = zip(*dates_stats[0]['values']) active_day = [xy[0][xy[1].index(max(xy[1]))], max(xy[1])] active_day[0] = datetime.fromtimestamp(active_day[0]/1000).strftime('%A') body = render_template('/account/email/weeklystats.md', project=project, dates_stats=dates_stats, hours_stats=hours_stats, users_stats=users_stats, n_completed_tasks=n_completed_tasks, active_day=active_day, config=current_app.config) html = render_template('/account/email/weeklystats.html', project=project, dates_stats=dates_stats, hours_stats=hours_stats, users_stats=users_stats, active_day=active_day, n_completed_tasks=n_completed_tasks, config=current_app.config) mail_dict = dict(recipients=[project.owner.email_addr], subject=subject, body=body, html=html) job = dict(name=send_mail, args=[mail_dict], kwargs={}, timeout=timeout, queue='high') enqueue_job(job)
def send_weekly_stats_project(project_id): from pybossa.cache.project_stats import update_stats, get_stats from pybossa.core import project_repo from datetime import datetime project = project_repo.get(project_id) if project.owner.subscribed is False: return "Owner does not want updates by email" update_stats(project_id) dates_stats, hours_stats, users_stats = get_stats(project_id, geo=True, period='1 week') subject = "Weekly Update: %s" % project.name timeout = current_app.config.get('TIMEOUT') # Max number of completed tasks n_completed_tasks = 0 xy = zip(*dates_stats[3]['values']) n_completed_tasks = max(xy[1]) # Most active day xy = zip(*dates_stats[0]['values']) active_day = [xy[0][xy[1].index(max(xy[1]))], max(xy[1])] active_day[0] = datetime.fromtimestamp(active_day[0] / 1000).strftime('%A') body = render_template('/account/email/weeklystats.md', project=project, dates_stats=dates_stats, hours_stats=hours_stats, users_stats=users_stats, n_completed_tasks=n_completed_tasks, active_day=active_day, config=current_app.config) html = render_template('/account/email/weeklystats.html', project=project, dates_stats=dates_stats, hours_stats=hours_stats, users_stats=users_stats, active_day=active_day, n_completed_tasks=n_completed_tasks, config=current_app.config) mail_dict = dict(recipients=[project.owner.email_addr], subject=subject, body=body, html=html) job = dict(name=send_mail, args=[mail_dict], kwargs={}, timeout=timeout, queue='high') enqueue_job(job)
def progress(short_name): """Return progress for each volume and template.""" category = project_repo.get_category_by(short_name=short_name) if not category: # pragma: no cover abort(404) tmpl_index = {t['id']: t for t in category.info.get('templates', [])} vol_index = {v['id']: v for v in category.info.get('volumes', [])} data = {v_id: {t_id: None for t_id in tmpl_index} for v_id in vol_index} projects = project_repo.filter_by(category_id=category.id) for project in projects: ps = project_stats.get_stats(project.id, full=True) try: row = data[project.info['volume_id']] row[project.info['template_id']] = ps.overall_progress except KeyError: continue # Replace IDs with names and flatten flat_data = [] for vol_id in data: row = {'Volume': vol_index[vol_id]['name']} for tmpl_id in data[vol_id]: tmpl_name = tmpl_index[tmpl_id]['name'] if tmpl_name == 'Volume': tmpl_name = '_Volume' row[tmpl_name] = data[vol_id][tmpl_id] flat_data.append(row) if request.args.get('csv'): df = pandas.DataFrame(flat_data) df.set_index('Volume', inplace=True) csv = df.to_csv(encoding='utf8') response = dict(progress=csv) return handle_content_type(response) response = dict(progress=flat_data) return handle_content_type(response)
def test_03_stats(self): """Test STATS stats method works""" self.prepare_data() today = str(datetime.date.today()) hour = int(datetime.datetime.utcnow().strftime('%H')) date_ms = time.mktime(time.strptime(today, "%Y-%m-%d")) * 1000 anon = 0 auth = 0 tr1 = TaskRunFactory.create(task=self.project.tasks[0]) tr2 = TaskRunFactory.create(task=self.project.tasks[1]) user = user_repo.get(tr1.user_id) user.restrict = True user_repo.update(user) stats.update_stats(self.project.id) dates_stats, hours_stats, user_stats = stats.get_stats(self.project.id) for item in dates_stats: if item['label'] == 'Anon + Auth': assert item['values'][-1][0] == date_ms, item['values'][0][0] assert item['values'][-1][ 1] == 10, "There should be 10 answers" if item['label'] == 'Anonymous': assert item['values'][-1][0] == date_ms, item['values'][0][0] anon = item['values'][-1][1] if item['label'] == 'Authenticated': assert item['values'][-1][0] == date_ms, item['values'][0][0] auth = item['values'][-1][1] if item['label'] == 'Total Tasks': assert item['values'][-1][0] == date_ms, item['values'][0][0] assert item['values'][-1][1] == 4, "There should be 4 tasks" if item['label'] == 'Expected Answers': assert item['values'][0][0] == date_ms, item['values'][0][0] for i in item['values']: assert i[1] == 100, "Each date should have 100 answers" assert item['values'][0][ 1] == 100, "There should be 10 answers" assert auth + anon == 10, "date stats sum of auth and anon should be 10" max_hours = 0 for item in hours_stats: if item['label'] == 'Anon + Auth': max_hours = item['max'] print(item) assert item['max'] == 10, item['max'] assert item['max'] == 10, "Max hours value should be 10" for i in item['values']: if i[0] == hour: assert i[1] == 10, "There should be 10 answers" assert i[2] == 5, "The size of the bubble should be 5" else: assert i[1] == 0, "There should be 0 answers" assert i[2] == 0, "The size of the buggle should be 0" if item['label'] == 'Anonymous': anon = item['max'] for i in item['values']: if i[0] == hour: assert i[1] == anon, "There should be anon answers" assert i[2] == ( anon * 5 ) / max_hours, "The size of the bubble should be 5" else: assert i[1] == 0, "There should be 0 answers" assert i[2] == 0, "The size of the buggle should be 0" if item['label'] == 'Authenticated': auth = item['max'] for i in item['values']: if i[0] == hour: assert i[1] == auth, "There should be anon answers" assert i[2] == ( auth * 5 ) / max_hours, "The size of the bubble should be 5" else: assert i[1] == 0, "There should be 0 answers" assert i[2] == 0, "The size of the buggle should be 0" assert auth + anon == 10, "date stats sum of auth and anon should be 8" err_msg = "user stats sum of auth and anon should be 7" assert user_stats['n_anon'] + user_stats['n_auth'] == 7, err_msg for u in user_stats['auth']['top5']: assert u['restrict'] is False, u
def test_03_stats(self): """Test STATS stats method works""" today = unicode(datetime.date.today()) hour = int(datetime.datetime.utcnow().strftime('%H')) date_ms = time.mktime(time.strptime(today, "%Y-%m-%d")) * 1000 anon = 0 auth = 0 TaskRunFactory.create(task=self.project.tasks[0]) TaskRunFactory.create(task=self.project.tasks[1]) dates_stats, hours_stats, user_stats = stats.get_stats(self.project.id) for item in dates_stats: if item['label'] == 'Anon + Auth': assert item['values'][0][0] == date_ms, item['values'][0][0] assert item['values'][0][1] == 10, "There should be 10 answers" if item['label'] == 'Anonymous': assert item['values'][0][0] == date_ms, item['values'][0][0] anon = item['values'][0][1] if item['label'] == 'Authenticated': assert item['values'][0][0] == date_ms, item['values'][0][0] auth = item['values'][0][1] if item['label'] == 'Total Tasks': assert item['values'][0][0] == date_ms, item['values'][0][0] assert item['values'][0][1] == 4, "There should be 4 tasks" if item['label'] == 'Expected Answers': assert item['values'][0][0] == date_ms, item['values'][0][0] for i in item['values']: assert i[1] == 100, "Each date should have 100 answers" assert item['values'][0][1] == 100, "There should be 10 answers" assert auth + anon == 10, "date stats sum of auth and anon should be 10" max_hours = 0 for item in hours_stats: if item['label'] == 'Anon + Auth': max_hours = item['max'] print item assert item['max'] == 10, item['max'] assert item['max'] == 10, "Max hours value should be 10" for i in item['values']: if i[0] == hour: assert i[1] == 10, "There should be 10 answers" assert i[2] == 5, "The size of the bubble should be 5" else: assert i[1] == 0, "There should be 0 answers" assert i[2] == 0, "The size of the buggle should be 0" if item['label'] == 'Anonymous': anon = item['max'] for i in item['values']: if i[0] == hour: assert i[1] == anon, "There should be anon answers" assert i[2] == (anon * 5) / max_hours, "The size of the bubble should be 5" else: assert i[1] == 0, "There should be 0 answers" assert i[2] == 0, "The size of the buggle should be 0" if item['label'] == 'Authenticated': auth = item['max'] for i in item['values']: if i[0] == hour: assert i[1] == auth, "There should be anon answers" assert i[2] == (auth * 5) / max_hours, "The size of the bubble should be 5" else: assert i[1] == 0, "There should be 0 answers" assert i[2] == 0, "The size of the buggle should be 0" assert auth + anon == 10, "date stats sum of auth and anon should be 8" err_msg = "user stats sum of auth and anon should be 7" assert user_stats['n_anon'] + user_stats['n_auth'] == 7, err_msg
def test_query_projectstats(self): """Test API query for project stats endpoint works""" project_stats = [] projects = ProjectFactory.create_batch(3) for project in projects: for task in TaskFactory.create_batch(4, project=project, n_answers=3): TaskRunFactory.create(task=task) stats.update_stats(project.id) ps = stats.get_stats(project.id, full=True) project_stats.append(ps) extra_stat_types = ['hours_stats', 'dates_stats', 'users_stats'] # As anon url = '/api/projectstats' res = self.app_get_json(url) data = json.loads(res.data) assert len(data) == 3, data # Limits res = self.app.get(url + "?limit=1") data = json.loads(res.data) assert len(data) == 1, data # Keyset pagination res = self.app.get(url + '?limit=1&last_id=' + str(projects[1].id)) data = json.loads(res.data) assert len(data) == 1, len(data) assert data[0]['id'] == project.id # Errors res = self.app.get(url + "?something") err = json.loads(res.data) err_msg = "AttributeError exception should be raised" res.status_code == 415, err_msg assert res.status_code == 415, err_msg assert err['action'] == 'GET', err_msg assert err['status'] == 'failed', err_msg assert err['exception_cls'] == 'AttributeError', err_msg # Desc filter url = "/api/projectstats?orderby=wrongattribute" res = self.app.get(url) data = json.loads(res.data) err_msg = "It should be 415." assert data['status'] == 'failed', data assert data['status_code'] == 415, data assert 'has no attribute' in data['exception_msg'], data # Order by url = "/api/projectstats?orderby=id" res = self.app.get(url) data = json.loads(res.data) err_msg = "It should get the last item first." ps_by_id = sorted(project_stats, key=lambda x: x.id, reverse=False) for i in range(len(project_stats)): assert ps_by_id[i].id == data[i]['id'] # Desc filter url = "/api/projectstats?orderby=id&desc=true" res = self.app.get(url) data = json.loads(res.data) err_msg = "It should get the last item first." ps_by_id = sorted(project_stats, key=lambda x: x.id, reverse=True) for i in range(len(project_stats)): assert ps_by_id[i].id == data[i]['id'] # Without full filter url = "/api/projectstats" res = self.app.get(url) data = json.loads(res.data) err_msg = "It should not return the full stats." extra = [row['info'].get(_type) for _type in extra_stat_types for row in data if row['info'].get(_type)] assert not extra # With full filter url = "/api/projectstats?full=1" res = self.app.get(url) data = json.loads(res.data) err_msg = "It should return full stats." for i, row in enumerate(data): for _type in extra_stat_types: assert row['info'][_type] == project_stats[i].info[_type]
def prepare_stats(self): project = ProjectFactory.create() stats.update_stats(project.id) return stats.get_stats(project.id, full=True)
def test_query_projectstats(self): """Test API query for project stats endpoint works""" project_stats = [] projects = ProjectFactory.create_batch(3) for project in projects: for task in TaskFactory.create_batch(4, project=project, n_answers=3): TaskRunFactory.create(task=task) stats.update_stats(project.id) ps = stats.get_stats(project.id, full=True) project_stats.append(ps) extra_stat_types = ['hours_stats', 'dates_stats', 'users_stats'] # As anon url = '/api/projectstats' res = self.app_get_json(url) data = json.loads(res.data) assert len(data) == 3, data # Limits res = self.app.get(url + "?limit=1") data = json.loads(res.data) assert len(data) == 1, data # Keyset pagination res = self.app.get(url + '?limit=1&last_id=' + str(projects[1].id)) data = json.loads(res.data) assert len(data) == 1, len(data) assert data[0]['id'] == project.id # Errors res = self.app.get(url + "?something") err = json.loads(res.data) err_msg = "AttributeError exception should be raised" res.status_code == 415, err_msg assert res.status_code == 415, err_msg assert err['action'] == 'GET', err_msg assert err['status'] == 'failed', err_msg assert err['exception_cls'] == 'AttributeError', err_msg # Desc filter url = "/api/projectstats?orderby=wrongattribute" res = self.app.get(url) data = json.loads(res.data) err_msg = "It should be 415." assert data['status'] == 'failed', data assert data['status_code'] == 415, data assert 'has no attribute' in data['exception_msg'], data # Order by url = "/api/projectstats?orderby=id" res = self.app.get(url) data = json.loads(res.data) err_msg = "It should get the last item first." ps_by_id = sorted(project_stats, key=lambda x: x.id, reverse=False) for i in range(len(project_stats)): assert ps_by_id[i].id == data[i]['id'] # Desc filter url = "/api/projectstats?orderby=id&desc=true" res = self.app.get(url) data = json.loads(res.data) err_msg = "It should get the last item first." ps_by_id = sorted(project_stats, key=lambda x: x.id, reverse=True) for i in range(len(project_stats)): assert ps_by_id[i].id == data[i]['id'] # Without full filter url = "/api/projectstats" res = self.app.get(url) data = json.loads(res.data) err_msg = "It should not return the full stats." extra = [ row['info'].get(_type) for _type in extra_stat_types for row in data if row['info'].get(_type) ] assert not extra # With full filter url = "/api/projectstats?full=1" res = self.app.get(url) data = json.loads(res.data) err_msg = "It should return full stats." for i, row in enumerate(data): for _type in extra_stat_types: assert row['info'][_type] == project_stats[i].info[_type]
def prepare_stats(self, n=1): project = ProjectFactory.create() TaskFactory.create_batch(n, project=project) stats.update_stats(project.id) return stats.get_stats(project.id, full=True)