def test_get_returns_required_fields(self):
"""Test CACHE PROJECTS get returns the required info
about each project"""
fields = (
"id",
"name",
"short_name",
"info",
"created",
"description",
"last_activity",
"last_activity_raw",
"overall_progress",
"n_tasks",
"n_volunteers",
"owner",
"info",
"updated",
)
project = self.create_project_with_tasks(1, 0)
retrieved_project = cached_projects.get(project.category.short_name)[0]
for field in fields:
assert retrieved_project.has_key(field), "%s not in project info" % field
def home():
"""Render home page with the cached projects and users."""
page = 1
per_page = current_app.config.get('APPS_PER_PAGE')
if per_page is None: # pragma: no cover
per_page = 5
d = {'top_projects': cached_projects.get_top(),
'top_users': None}
# Get all the categories with projects
categories = cached_cat.get_used()
d['categories'] = categories
d['categories_projects'] = {}
for c in categories:
tmp_projects = cached_projects.get(c['short_name'], page, per_page)
d['categories_projects'][c['short_name']] = rank(tmp_projects)
# Add featured
tmp_projects = cached_projects.get_featured('featured', page, per_page)
if len(tmp_projects) > 0:
featured = Category(name='Featured', short_name='featured')
d['categories'].insert(0, featured)
d['categories_projects']['featured'] = rank(tmp_projects)
if (current_app.config['ENFORCE_PRIVACY']
and current_user.is_authenticated()):
if current_user.admin:
d['top_users'] = cached_users.get_top()
if not current_app.config['ENFORCE_PRIVACY']:
d['top_users'] = cached_users.get_top()
return render_template('/home/index.html', **d)
def test_get_only_return_published(self):
"""Test CACHE PROJECTS get returns only published projects"""
project = ProjectFactory.create(published=True)
ProjectFactory.create(category=project.category, published=False)
projects = cached_projects.get(project.category.short_name)
assert len(projects) is 1, projects
def test_get_not_returns_hidden_projects(self):
"""Test CACHE PROJECTS get does not return hidden projects"""
project = self.create_project_with_contributors(1, 0, hidden=1)
projects = cached_projects.get(project.category.short_name)
assert len(projects) is 0, projects
def test_get_dont_return_projects_with_password(self):
"""Test CACHE PROJECTS get does not return projects with a password"""
project = ProjectFactory.create(published=True, info={'passwd_hash': '2'})
ProjectFactory.create(category=project.category, published=True)
projects = cached_projects.get(project.category.short_name)
assert len(projects) is 1, projects
def test_get_category(self):
"""Test CACHE PROJECTS get returns projects from given category"""
project = self.create_project_with_tasks(1, 0)
projects = cached_projects.get(project.category.short_name)
assert len(projects) is 1, projects
def api():
"""Render help/api page."""
categories = cached_cat.get_used()
projects = cached_projects.get(categories[0]["short_name"])
if len(projects) > 0:
project_id = choice(projects)["id"]
else: # pragma: no cover
project_id = None
return render_template("help/api.html", title="Help: API", project_id=project_id)
def test_get_only_returns_projects_from_category(self):
"""Test CACHE PROJECTS get returns only projects from required category"""
project = ProjectFactory.create(published=True)
ProjectFactory.create(published=True)
projects = cached_projects.get(project.category.short_name)
assert len(projects) is 1, projects
def test_get_only_returns_category_projects(self):
"""Test CACHE PROJECTS get returns only projects from required category"""
project = self.create_project_with_tasks(1, 0)
#create a non published project too
ProjectFactory.create()
projects = cached_projects.get(project.category.short_name)
assert len(projects) is 1, projects
def api():
"""Render help/api page."""
categories = cached_cat.get_used()
apps = cached_projects.get(categories[0]['short_name'])
if len(apps) > 0:
project_id = choice(apps)['id']
else: # pragma: no cover
project_id = None
return render_template('help/api.html', title="Help: API",
project_id=project_id)
def test_get_not_returns_draft_projects(self):
"""Test CACHE PROJECTS get does not return draft (non-published) projects"""
project = self.create_project_with_contributors(1, 0)
# Create a project wothout presenter
ProjectFactory.create(info={}, category=project.category)
projects = cached_projects.get(project.category.short_name)
assert len(projects) is 1, projects
def api():
"""Render help/api page."""
categories = cached_cat.get_used()
projects = cached_projects.get(categories[0]['short_name'])
if len(projects) > 0:
project_id = choice(projects)['id']
else: # pragma: no cover
project_id = None
response = dict(template='help/api.html',
title="Help: API",
project_id=project_id)
return handle_content_type(response)
def test_get_returns_required_fields(self):
"""Test CACHE PROJECTS get returns the required info
about each project"""
fields = ('id', 'name', 'short_name', 'info', 'created', 'description',
'last_activity', 'last_activity_raw', 'overall_progress',
'n_tasks', 'n_volunteers', 'owner', 'info', 'updated')
project = self.create_project_with_tasks(1, 0)
retrieved_project = cached_projects.get(project.category.short_name)[0]
for field in fields:
assert retrieved_project.has_key(field), "%s not in project info" % field
def warm_cache():
'''Warm cache'''
# Disable cache, so we can refresh the data in Redis
os.environ['PYBOSSA_REDIS_CACHE_DISABLED'] = '1'
# Cache 3 pages
apps_cached = []
pages = range(1, 4)
with app.app_context():
import pybossa.cache.projects as cached_apps
import pybossa.cache.categories as cached_cat
import pybossa.cache.users as cached_users
import pybossa.cache.project_stats as stats
def warm_app(id, short_name, featured=False):
if id not in apps_cached:
cached_apps.get_app(short_name)
cached_apps.n_tasks(id)
n_task_runs = cached_apps.n_task_runs(id)
cached_apps.overall_progress(id)
cached_apps.last_activity(id)
cached_apps.n_completed_tasks(id)
cached_apps.n_volunteers(id)
if n_task_runs >= 1000 or featured:
print "Getting stats for %s as it has %s task runs" % (short_name, n_task_runs)
stats.get_stats(id, app.config.get('GEO'))
apps_cached.append(id)
# Cache top projects
apps = cached_apps.get_top()
for a in apps:
warm_app(a['id'], a['short_name'])
for page in pages:
apps = cached_apps.get_featured('featured', page,
app.config['APPS_PER_PAGE'])
for a in apps:
warm_app(a['id'], a['short_name'], featured=True)
# Categories
categories = cached_cat.get_used()
for c in categories:
for page in pages:
apps = cached_apps.get(c['short_name'],
page,
app.config['APPS_PER_PAGE'])
for a in apps:
warm_app(a['id'], a['short_name'])
# Users
cached_users.get_leaderboard(app.config['LEADERBOARD'], 'anonymous')
cached_users.get_top()
def index():
"""Return the Data page."""
categories = cached_cat.get_all()
projects = {}
for c in categories:
n_projects = cached_projects.n_count(category=c.short_name)
projects[c.short_name] = cached_projects.get(category=c.short_name,
page=1,
per_page=n_projects)
for p in projects[c.short_name]:
p['n_task_runs'] = cached_projects.n_task_runs(p['id'])
p['n_results'] = cached_projects.n_results(p['id'])
return render_template('/index.html', projects=projects,
categories=categories, title="Data")
def test_get_returns_required_fields(self):
"""Test CACHE PROJECTS get returns the required info
about each project"""
fields = ('id', 'name', 'short_name', 'info', 'created', 'description',
'last_activity', 'last_activity_raw', 'overall_progress',
'n_tasks', 'n_volunteers', 'owner', 'info', 'updated')
project = ProjectFactory.create(published=True)
retrieved_project = cached_projects.get(project.category.short_name)[0]
for field in fields:
assert retrieved_project.has_key(
field), "%s not in project info" % field
def warm_cache():
'''Warm cache'''
# Disable cache, so we can refresh the data in Redis
os.environ['PYBOSSA_REDIS_CACHE_DISABLED'] = '1'
# Cache 3 pages
apps_cached = []
pages = range(1, 4)
with app.app_context():
import pybossa.cache.projects as cached_apps
import pybossa.cache.categories as cached_cat
import pybossa.cache.users as cached_users
import pybossa.cache.project_stats as stats
def warm_app(id, short_name, featured=False):
if id not in apps_cached:
cached_apps.get_app(short_name)
cached_apps.n_tasks(id)
n_task_runs = cached_apps.n_task_runs(id)
cached_apps.overall_progress(id)
cached_apps.last_activity(id)
cached_apps.n_completed_tasks(id)
cached_apps.n_volunteers(id)
if n_task_runs >= 1000 or featured:
print "Getting stats for %s as it has %s task runs" % (
short_name, n_task_runs)
stats.get_stats(id)
apps_cached.append(id)
# Cache top projects
apps = cached_apps.get_top()
for a in apps:
warm_app(a['id'], a['short_name'])
for page in pages:
apps = cached_apps.get_featured('featured', page,
app.config['APPS_PER_PAGE'])
for a in apps:
warm_app(a['id'], a['short_name'], featured=True)
# Categories
categories = cached_cat.get_used()
for c in categories:
for page in pages:
apps = cached_apps.get(c['short_name'], page,
app.config['APPS_PER_PAGE'])
for a in apps:
warm_app(a['id'], a['short_name'])
# Users
cached_users.get_leaderboard(app.config['LEADERBOARD'])
def featured(project_id=None):
"""List featured projects of PYBOSSA."""
try:
if request.method == 'GET':
categories = cached_cat.get_all()
projects = {}
for c in categories:
n_projects = cached_projects.n_count(category=c.short_name)
projects[c.short_name] = cached_projects.get(
category=c.short_name,
page=1,
per_page=n_projects)
response = dict(template = '/admin/projects.html',
projects=projects,
categories=categories,
form=dict(csrf=generate_csrf()))
return handle_content_type(response)
else:
project = project_repo.get(project_id)
if project:
ensure_authorized_to('update', project)
if request.method == 'POST':
if project.featured is True:
msg = "Project.id %s already featured" % project_id
return format_error(msg, 415)
cached_projects.reset()
project.featured = True
project_repo.update(project)
return json.dumps(project.dictize())
if request.method == 'DELETE':
if project.featured is False:
msg = 'Project.id %s is not featured' % project_id
return format_error(msg, 415)
cached_projects.reset()
project.featured = False
project_repo.update(project)
return json.dumps(project.dictize())
else:
msg = 'Project.id %s not found' % project_id
return format_error(msg, 404)
except Exception as e: # pragma: no cover
current_app.logger.error(e)
return abort(500)
def featured(project_id=None):
"""List featured projects of PYBOSSA."""
try:
if request.method == 'GET':
categories = cached_cat.get_all()
projects = {}
for c in categories:
n_projects = cached_projects.n_count(category=c.short_name)
projects[c.short_name] = cached_projects.get(
category=c.short_name,
page=1,
per_page=n_projects)
response = dict(template = '/admin/projects.html',
projects=projects,
categories=categories,
form=dict(csrf=generate_csrf()))
return handle_content_type(response)
else:
project = project_repo.get(project_id)
if project:
ensure_authorized_to('update', project)
if request.method == 'POST':
if project.featured is True:
msg = "Project.id %s already featured" % project_id
return format_error(msg, 415)
cached_projects.reset()
project.featured = True
project_repo.update(project)
return json.dumps(project.dictize())
if request.method == 'DELETE':
if project.featured is False:
msg = 'Project.id %s is not featured' % project_id
return format_error(msg, 415)
cached_projects.reset()
project.featured = False
project_repo.update(project)
return json.dumps(project.dictize())
else:
msg = 'Project.id %s not found' % project_id
return format_error(msg, 404)
except Exception as e: # pragma: no cover
current_app.logger.error(e)
return abort(500)
def _export_category_results_as_geoJSON(category_name):
from pybossa.cache import projects as cached_projects
from pybossa.exporter.json_export import JsonExporter
import json
import pandas as pd
import numpy as np
geotagx_json_exporter = JsonExporter()
max_number_of_exportable_projects = 15
projects_in_category = cached_projects.get(category_name, page=1, per_page=max_number_of_exportable_projects)
task_runs = []
task_runs_info = []
project_name_id_mapping = {}
project_id_name_mapping = {}
project_question_type_mapping = {}
project_question_question_text_mapping = {}
for project in projects_in_category:
short_name = project['short_name']
project_id_name_mapping[project['id']] = project['short_name']
project_name_id_mapping[project['short_name']] = project['id']
# Check if it a supported geotagx project whose schema we know
if 'GEOTAGX_SUPPORTED_PROJECTS_SCHEMA' in current_app.config.keys() \
and short_name in current_app.config['GEOTAGX_SUPPORTED_PROJECTS_SCHEMA'].keys():
##Read the project schema and store the respective questions and their types
for _question in current_app.config['GEOTAGX_SUPPORTED_PROJECTS_SCHEMA'][short_name]['questions']:
project_question_type_mapping[unicode(short_name+"::"+_question['answer']['saved_as'])] = _question['type']
project_question_question_text_mapping[unicode(short_name+"::"+_question['answer']['saved_as']+"::question_text")] = _question['title']
#Only export results of known GEOTAGX projects that are created with `geotagx-project-template`
task_runs_generator = geotagx_json_exporter.gen_json("task_run", project['id'])
_task_runs = ""
for task_run_c in task_runs_generator:
_task_runs += task_run_c
task_runs = task_runs + json.loads(_task_runs)
def extract_geotagx_info(json):
"""Returns a list of only info objects of the task_run"""
exploded_json = []
for item in json:
item['info']['project_id'] = item['project_id']
exploded_json.append(item['info'])
return exploded_json
def _summarize_geolocations(geolocation_responses):
"""
TODO :: Add different geo-summarization methods (ConvexHull, Centroid, etc)
"""
responses = []
for response in geolocation_responses:
if type(response) == type([]):
responses.append(response)
return responses
"""
Changes projection to WGS84 projection from WebMercator projection
so that most geojson renderers support it out of the box
Inspired by : http://www.gal-systems.com/2011/07/convert-coordinates-between-web.html
"""
def _project_coordinate_from_webmercator_toWGS84(coordinates):
import math
mercatorX_lon = coordinates[0]
mercatorY_lat = coordinates[1]
if math.fabs(mercatorX_lon) < 180 and math.fabs(mercatorY_lat) < 90:
return False, False
if ((math.fabs(mercatorX_lon) > 20037508.3427892) or (math.fabs(mercatorY_lat) > 20037508.3427892)):
return False, False
x = mercatorX_lon
y = mercatorY_lat
num3 = x / 6378137.0
num4 = num3 * 57.295779513082323
num5 = math.floor(float((num4 + 180.0) / 360.0))
num6 = num4 - (num5 * 360.0)
num7 = 1.5707963267948966 - (2.0 * math.atan(math.exp((-1.0 * y) / 6378137.0)));
mercatorX_lon = num6
mercatorY_lat = num7 * 57.295779513082323
return mercatorX_lon, mercatorY_lat
"""
Changes the projection of the multi_polygon object to WGS84 from WebMercator
"""
def _project_geosummary_from_webmercator_to_WGS84(multi_polygon):
_multi_polygon = []
for polygon in multi_polygon:
_polygon = []
for coordinates in polygon:
try:
_x, _y = _project_coordinate_from_webmercator_toWGS84(coordinates)
if _x and _y:
_polygon.append([_x, _y])
except:
pass # Pass Silentily if there is some error in the input
_multi_polygon.append(_polygon)
return _multi_polygon
def _build_geo_json(geolocation_responses):
geoJSON = {}
geoJSON['type'] = "FeatureCollection"
geoJSON['features'] = []
for response in geolocation_responses:
if response['_geotagx_geolocation_key']:
geo_summary = response[response['_geotagx_geolocation_key']]
_feature = {}
_feature['type'] = "Feature"
_feature['geometry'] = {}
_feature['geometry']['type'] = "MultiPolygon"
_feature['geometry']['coordinates'] = \
[_project_geosummary_from_webmercator_to_WGS84(geo_summary['geo_summary'])]
del response[response['_geotagx_geolocation_key']]
del response['_geotagx_geolocation_key']
_feature['properties'] = response
#Neglect responses with no coordinate labels
if _feature['geometry']['coordinates'] != [[]]:
geoJSON['features'].append(_feature)
return geoJSON
task_runs_info = extract_geotagx_info(task_runs)
task_runs_info = pd.read_json(json.dumps(task_runs_info))
summary_dict = {}
for img_url in task_runs_info['img'].unique():
per_url_data = task_runs_info[task_runs_info['img'] == img_url]
for project_id in np.unique(per_url_data['project_id'].values):
per_summary_dict = {}
per_summary_dict['_geotagx_geolocation_key'] = False
if img_url in summary_dict.keys():
per_summary_dict = summary_dict[img_url]
per_summary_dict['GEOTAGX_IMAGE_URL'] = img_url
per_url_data_project_slice = per_url_data[per_url_data['project_id'] == project_id]
for key in per_url_data_project_slice.keys():
namespaced_key = project_id_name_mapping[project_id]+"::"+key
if key not in ['img', 'isMigrated', 'son_app_id', 'task_id', 'project_id']:
if namespaced_key in project_question_type_mapping.keys():
if project_question_type_mapping[namespaced_key] == u"geotagging":
per_summary_dict['_geotagx_geolocation_key'] = namespaced_key
per_summary_dict[namespaced_key] = {'geo_summary' : _summarize_geolocations(per_url_data_project_slice[key].values)}
else:
per_summary_dict[namespaced_key] = {'answer_summary':dict(per_url_data_project_slice[key].value_counts())}
per_summary_dict[namespaced_key]['question_text'] = project_question_question_text_mapping[unicode(namespaced_key+"::question_text")]
elif key == u"img":
per_summary_dict[project_id_name_mapping[project_id]+"::GEOTAGX_TOTAL"] = len(per_url_data_project_slice)
summary_dict[img_url] = per_summary_dict
geo_json = _build_geo_json(summary_dict.values())
return jsonify(geo_json)
