def test_get_returns_required_fields(self): """Test CACHE PROJECTS get returns the required info about each project""" fields = ( "id", "name", "short_name", "info", "created", "description", "last_activity", "last_activity_raw", "overall_progress", "n_tasks", "n_volunteers", "owner", "info", "updated", ) project = self.create_project_with_tasks(1, 0) retrieved_project = cached_projects.get(project.category.short_name)[0] for field in fields: assert retrieved_project.has_key(field), "%s not in project info" % field
def home(): """Render home page with the cached projects and users.""" page = 1 per_page = current_app.config.get('APPS_PER_PAGE') if per_page is None: # pragma: no cover per_page = 5 d = {'top_projects': cached_projects.get_top(), 'top_users': None} # Get all the categories with projects categories = cached_cat.get_used() d['categories'] = categories d['categories_projects'] = {} for c in categories: tmp_projects = cached_projects.get(c['short_name'], page, per_page) d['categories_projects'][c['short_name']] = rank(tmp_projects) # Add featured tmp_projects = cached_projects.get_featured('featured', page, per_page) if len(tmp_projects) > 0: featured = Category(name='Featured', short_name='featured') d['categories'].insert(0, featured) d['categories_projects']['featured'] = rank(tmp_projects) if (current_app.config['ENFORCE_PRIVACY'] and current_user.is_authenticated()): if current_user.admin: d['top_users'] = cached_users.get_top() if not current_app.config['ENFORCE_PRIVACY']: d['top_users'] = cached_users.get_top() return render_template('/home/index.html', **d)
def test_get_only_return_published(self): """Test CACHE PROJECTS get returns only published projects""" project = ProjectFactory.create(published=True) ProjectFactory.create(category=project.category, published=False) projects = cached_projects.get(project.category.short_name) assert len(projects) is 1, projects
def test_get_not_returns_hidden_projects(self): """Test CACHE PROJECTS get does not return hidden projects""" project = self.create_project_with_contributors(1, 0, hidden=1) projects = cached_projects.get(project.category.short_name) assert len(projects) is 0, projects
def test_get_dont_return_projects_with_password(self): """Test CACHE PROJECTS get does not return projects with a password""" project = ProjectFactory.create(published=True, info={'passwd_hash': '2'}) ProjectFactory.create(category=project.category, published=True) projects = cached_projects.get(project.category.short_name) assert len(projects) is 1, projects
def test_get_category(self): """Test CACHE PROJECTS get returns projects from given category""" project = self.create_project_with_tasks(1, 0) projects = cached_projects.get(project.category.short_name) assert len(projects) is 1, projects
def api(): """Render help/api page.""" categories = cached_cat.get_used() projects = cached_projects.get(categories[0]["short_name"]) if len(projects) > 0: project_id = choice(projects)["id"] else: # pragma: no cover project_id = None return render_template("help/api.html", title="Help: API", project_id=project_id)
def test_get_only_returns_projects_from_category(self): """Test CACHE PROJECTS get returns only projects from required category""" project = ProjectFactory.create(published=True) ProjectFactory.create(published=True) projects = cached_projects.get(project.category.short_name) assert len(projects) is 1, projects
def test_get_only_returns_category_projects(self): """Test CACHE PROJECTS get returns only projects from required category""" project = self.create_project_with_tasks(1, 0) #create a non published project too ProjectFactory.create() projects = cached_projects.get(project.category.short_name) assert len(projects) is 1, projects
def api(): """Render help/api page.""" categories = cached_cat.get_used() apps = cached_projects.get(categories[0]['short_name']) if len(apps) > 0: project_id = choice(apps)['id'] else: # pragma: no cover project_id = None return render_template('help/api.html', title="Help: API", project_id=project_id)
def test_get_not_returns_draft_projects(self): """Test CACHE PROJECTS get does not return draft (non-published) projects""" project = self.create_project_with_contributors(1, 0) # Create a project wothout presenter ProjectFactory.create(info={}, category=project.category) projects = cached_projects.get(project.category.short_name) assert len(projects) is 1, projects
def api(): """Render help/api page.""" categories = cached_cat.get_used() projects = cached_projects.get(categories[0]['short_name']) if len(projects) > 0: project_id = choice(projects)['id'] else: # pragma: no cover project_id = None response = dict(template='help/api.html', title="Help: API", project_id=project_id) return handle_content_type(response)
def test_get_returns_required_fields(self): """Test CACHE PROJECTS get returns the required info about each project""" fields = ('id', 'name', 'short_name', 'info', 'created', 'description', 'last_activity', 'last_activity_raw', 'overall_progress', 'n_tasks', 'n_volunteers', 'owner', 'info', 'updated') project = self.create_project_with_tasks(1, 0) retrieved_project = cached_projects.get(project.category.short_name)[0] for field in fields: assert retrieved_project.has_key(field), "%s not in project info" % field
def warm_cache(): '''Warm cache''' # Disable cache, so we can refresh the data in Redis os.environ['PYBOSSA_REDIS_CACHE_DISABLED'] = '1' # Cache 3 pages apps_cached = [] pages = range(1, 4) with app.app_context(): import pybossa.cache.projects as cached_apps import pybossa.cache.categories as cached_cat import pybossa.cache.users as cached_users import pybossa.cache.project_stats as stats def warm_app(id, short_name, featured=False): if id not in apps_cached: cached_apps.get_app(short_name) cached_apps.n_tasks(id) n_task_runs = cached_apps.n_task_runs(id) cached_apps.overall_progress(id) cached_apps.last_activity(id) cached_apps.n_completed_tasks(id) cached_apps.n_volunteers(id) if n_task_runs >= 1000 or featured: print "Getting stats for %s as it has %s task runs" % (short_name, n_task_runs) stats.get_stats(id, app.config.get('GEO')) apps_cached.append(id) # Cache top projects apps = cached_apps.get_top() for a in apps: warm_app(a['id'], a['short_name']) for page in pages: apps = cached_apps.get_featured('featured', page, app.config['APPS_PER_PAGE']) for a in apps: warm_app(a['id'], a['short_name'], featured=True) # Categories categories = cached_cat.get_used() for c in categories: for page in pages: apps = cached_apps.get(c['short_name'], page, app.config['APPS_PER_PAGE']) for a in apps: warm_app(a['id'], a['short_name']) # Users cached_users.get_leaderboard(app.config['LEADERBOARD'], 'anonymous') cached_users.get_top()
def index(): """Return the Data page.""" categories = cached_cat.get_all() projects = {} for c in categories: n_projects = cached_projects.n_count(category=c.short_name) projects[c.short_name] = cached_projects.get(category=c.short_name, page=1, per_page=n_projects) for p in projects[c.short_name]: p['n_task_runs'] = cached_projects.n_task_runs(p['id']) p['n_results'] = cached_projects.n_results(p['id']) return render_template('/index.html', projects=projects, categories=categories, title="Data")
def test_get_returns_required_fields(self): """Test CACHE PROJECTS get returns the required info about each project""" fields = ('id', 'name', 'short_name', 'info', 'created', 'description', 'last_activity', 'last_activity_raw', 'overall_progress', 'n_tasks', 'n_volunteers', 'owner', 'info', 'updated') project = ProjectFactory.create(published=True) retrieved_project = cached_projects.get(project.category.short_name)[0] for field in fields: assert retrieved_project.has_key( field), "%s not in project info" % field
def warm_cache(): '''Warm cache''' # Disable cache, so we can refresh the data in Redis os.environ['PYBOSSA_REDIS_CACHE_DISABLED'] = '1' # Cache 3 pages apps_cached = [] pages = range(1, 4) with app.app_context(): import pybossa.cache.projects as cached_apps import pybossa.cache.categories as cached_cat import pybossa.cache.users as cached_users import pybossa.cache.project_stats as stats def warm_app(id, short_name, featured=False): if id not in apps_cached: cached_apps.get_app(short_name) cached_apps.n_tasks(id) n_task_runs = cached_apps.n_task_runs(id) cached_apps.overall_progress(id) cached_apps.last_activity(id) cached_apps.n_completed_tasks(id) cached_apps.n_volunteers(id) if n_task_runs >= 1000 or featured: print "Getting stats for %s as it has %s task runs" % ( short_name, n_task_runs) stats.get_stats(id) apps_cached.append(id) # Cache top projects apps = cached_apps.get_top() for a in apps: warm_app(a['id'], a['short_name']) for page in pages: apps = cached_apps.get_featured('featured', page, app.config['APPS_PER_PAGE']) for a in apps: warm_app(a['id'], a['short_name'], featured=True) # Categories categories = cached_cat.get_used() for c in categories: for page in pages: apps = cached_apps.get(c['short_name'], page, app.config['APPS_PER_PAGE']) for a in apps: warm_app(a['id'], a['short_name']) # Users cached_users.get_leaderboard(app.config['LEADERBOARD'])
def featured(project_id=None): """List featured projects of PYBOSSA.""" try: if request.method == 'GET': categories = cached_cat.get_all() projects = {} for c in categories: n_projects = cached_projects.n_count(category=c.short_name) projects[c.short_name] = cached_projects.get( category=c.short_name, page=1, per_page=n_projects) response = dict(template = '/admin/projects.html', projects=projects, categories=categories, form=dict(csrf=generate_csrf())) return handle_content_type(response) else: project = project_repo.get(project_id) if project: ensure_authorized_to('update', project) if request.method == 'POST': if project.featured is True: msg = "Project.id %s already featured" % project_id return format_error(msg, 415) cached_projects.reset() project.featured = True project_repo.update(project) return json.dumps(project.dictize()) if request.method == 'DELETE': if project.featured is False: msg = 'Project.id %s is not featured' % project_id return format_error(msg, 415) cached_projects.reset() project.featured = False project_repo.update(project) return json.dumps(project.dictize()) else: msg = 'Project.id %s not found' % project_id return format_error(msg, 404) except Exception as e: # pragma: no cover current_app.logger.error(e) return abort(500)
def _export_category_results_as_geoJSON(category_name): from pybossa.cache import projects as cached_projects from pybossa.exporter.json_export import JsonExporter import json import pandas as pd import numpy as np geotagx_json_exporter = JsonExporter() max_number_of_exportable_projects = 15 projects_in_category = cached_projects.get(category_name, page=1, per_page=max_number_of_exportable_projects) task_runs = [] task_runs_info = [] project_name_id_mapping = {} project_id_name_mapping = {} project_question_type_mapping = {} project_question_question_text_mapping = {} for project in projects_in_category: short_name = project['short_name'] project_id_name_mapping[project['id']] = project['short_name'] project_name_id_mapping[project['short_name']] = project['id'] # Check if it a supported geotagx project whose schema we know if 'GEOTAGX_SUPPORTED_PROJECTS_SCHEMA' in current_app.config.keys() \ and short_name in current_app.config['GEOTAGX_SUPPORTED_PROJECTS_SCHEMA'].keys(): ##Read the project schema and store the respective questions and their types for _question in current_app.config['GEOTAGX_SUPPORTED_PROJECTS_SCHEMA'][short_name]['questions']: project_question_type_mapping[unicode(short_name+"::"+_question['answer']['saved_as'])] = _question['type'] project_question_question_text_mapping[unicode(short_name+"::"+_question['answer']['saved_as']+"::question_text")] = _question['title'] #Only export results of known GEOTAGX projects that are created with `geotagx-project-template` task_runs_generator = geotagx_json_exporter.gen_json("task_run", project['id']) _task_runs = "" for task_run_c in task_runs_generator: _task_runs += task_run_c task_runs = task_runs + json.loads(_task_runs) def extract_geotagx_info(json): """Returns a list of only info objects of the task_run""" exploded_json = [] for item in json: item['info']['project_id'] = item['project_id'] exploded_json.append(item['info']) return exploded_json def _summarize_geolocations(geolocation_responses): """ TODO :: Add different geo-summarization methods (ConvexHull, Centroid, etc) """ responses = [] for response in geolocation_responses: if type(response) == type([]): responses.append(response) return responses """ Changes projection to WGS84 projection from WebMercator projection so that most geojson renderers support it out of the box Inspired by : http://www.gal-systems.com/2011/07/convert-coordinates-between-web.html """ def _project_coordinate_from_webmercator_toWGS84(coordinates): import math mercatorX_lon = coordinates[0] mercatorY_lat = coordinates[1] if math.fabs(mercatorX_lon) < 180 and math.fabs(mercatorY_lat) < 90: return False, False if ((math.fabs(mercatorX_lon) > 20037508.3427892) or (math.fabs(mercatorY_lat) > 20037508.3427892)): return False, False x = mercatorX_lon y = mercatorY_lat num3 = x / 6378137.0 num4 = num3 * 57.295779513082323 num5 = math.floor(float((num4 + 180.0) / 360.0)) num6 = num4 - (num5 * 360.0) num7 = 1.5707963267948966 - (2.0 * math.atan(math.exp((-1.0 * y) / 6378137.0))); mercatorX_lon = num6 mercatorY_lat = num7 * 57.295779513082323 return mercatorX_lon, mercatorY_lat """ Changes the projection of the multi_polygon object to WGS84 from WebMercator """ def _project_geosummary_from_webmercator_to_WGS84(multi_polygon): _multi_polygon = [] for polygon in multi_polygon: _polygon = [] for coordinates in polygon: try: _x, _y = _project_coordinate_from_webmercator_toWGS84(coordinates) if _x and _y: _polygon.append([_x, _y]) except: pass # Pass Silentily if there is some error in the input _multi_polygon.append(_polygon) return _multi_polygon def _build_geo_json(geolocation_responses): geoJSON = {} geoJSON['type'] = "FeatureCollection" geoJSON['features'] = [] for response in geolocation_responses: if response['_geotagx_geolocation_key']: geo_summary = response[response['_geotagx_geolocation_key']] _feature = {} _feature['type'] = "Feature" _feature['geometry'] = {} _feature['geometry']['type'] = "MultiPolygon" _feature['geometry']['coordinates'] = \ [_project_geosummary_from_webmercator_to_WGS84(geo_summary['geo_summary'])] del response[response['_geotagx_geolocation_key']] del response['_geotagx_geolocation_key'] _feature['properties'] = response #Neglect responses with no coordinate labels if _feature['geometry']['coordinates'] != [[]]: geoJSON['features'].append(_feature) return geoJSON task_runs_info = extract_geotagx_info(task_runs) task_runs_info = pd.read_json(json.dumps(task_runs_info)) summary_dict = {} for img_url in task_runs_info['img'].unique(): per_url_data = task_runs_info[task_runs_info['img'] == img_url] for project_id in np.unique(per_url_data['project_id'].values): per_summary_dict = {} per_summary_dict['_geotagx_geolocation_key'] = False if img_url in summary_dict.keys(): per_summary_dict = summary_dict[img_url] per_summary_dict['GEOTAGX_IMAGE_URL'] = img_url per_url_data_project_slice = per_url_data[per_url_data['project_id'] == project_id] for key in per_url_data_project_slice.keys(): namespaced_key = project_id_name_mapping[project_id]+"::"+key if key not in ['img', 'isMigrated', 'son_app_id', 'task_id', 'project_id']: if namespaced_key in project_question_type_mapping.keys(): if project_question_type_mapping[namespaced_key] == u"geotagging": per_summary_dict['_geotagx_geolocation_key'] = namespaced_key per_summary_dict[namespaced_key] = {'geo_summary' : _summarize_geolocations(per_url_data_project_slice[key].values)} else: per_summary_dict[namespaced_key] = {'answer_summary':dict(per_url_data_project_slice[key].value_counts())} per_summary_dict[namespaced_key]['question_text'] = project_question_question_text_mapping[unicode(namespaced_key+"::question_text")] elif key == u"img": per_summary_dict[project_id_name_mapping[project_id]+"::GEOTAGX_TOTAL"] = len(per_url_data_project_slice) summary_dict[img_url] = per_summary_dict geo_json = _build_geo_json(summary_dict.values()) return jsonify(geo_json)