def test_exporters_generates_zip(self, json_uploader, csv_uploader):
"""Test that CsvExporter and JsonExporter generate zip works."""
user = UserFactory.create(admin=True)
project = ProjectFactory.create(name='test_project')
task = TaskFactory.create(project=project)
task_run = TaskRunFactory.create(project=project, task=task)
csv_exporter = CsvExporter()
json_exporter = JsonExporter()
csv_exporter.pregenerate_zip_files(project)
call_csv_params = csv_uploader.upload_file.call_args_list
expected_csv_params = set(['1_project1_task_run_csv.zip', '1_project1_result_csv.zip', '1_project1_task_csv.zip'])
assert self._check_func_called_with_params(call_csv_params, expected_csv_params)
json_exporter.pregenerate_zip_files(project)
call_json_params = json_uploader.upload_file.call_args_list
expected_json_params = set(['1_project1_task_run_json.zip', '1_project1_result_json.zip', '1_project1_task_json.zip'])
assert self._check_func_called_with_params(call_json_params, expected_json_params)
def export_userdata(user_id, **kwargs):
from pybossa.core import user_repo, project_repo, task_repo, result_repo
from flask import current_app, url_for
json_exporter = JsonExporter()
user = user_repo.get(user_id)
user_data = user.dictize()
del user_data['passwd_hash']
projects = project_repo.filter_by(owner_id=user.id)
projects_data = [project.dictize() for project in projects]
taskruns = task_repo.filter_task_runs_by(user_id=user.id)
taskruns_data = [tr.dictize() for tr in taskruns]
pdf = json_exporter._make_zip(None, '', 'personal_data', user_data, user_id,
'personal_data.zip')
upf = json_exporter._make_zip(None, '', 'user_projects', projects_data, user_id,
'user_projects.zip')
ucf = json_exporter._make_zip(None, '', 'user_contributions', taskruns_data, user_id,
'user_contributions.zip')
upload_method = current_app.config.get('UPLOAD_METHOD')
if upload_method == 'local':
upload_method = 'uploads.uploaded_file'
personal_data_link = url_for(upload_method,
filename="user_%s/%s" % (user_id, pdf))
personal_projects_link = url_for(upload_method,
filename="user_%s/%s" % (user_id,
upf))
personal_contributions_link = url_for(upload_method,
filename="user_%s/%s" % (user_id,
ucf))
body = render_template('/account/email/exportdata.md',
user=user.dictize(),
personal_data_link=personal_data_link,
personal_projects_link=personal_projects_link,
personal_contributions_link=personal_contributions_link,
config=current_app.config)
html = render_template('/account/email/exportdata.html',
user=user.dictize(),
personal_data_link=personal_data_link,
personal_projects_link=personal_projects_link,
personal_contributions_link=personal_contributions_link,
config=current_app.config)
subject = 'Your personal data'
mail_dict = dict(recipients=[user.email_addr],
subject=subject,
body=body,
html=html)
send_mail(mail_dict)
def setup_exporter(app):
"""Setup exporter."""
global csv_exporter
global task_csv_exporter
global json_exporter
global task_json_exporter
global project_csv_exporter
from pybossa.exporter.csv_export import CsvExporter
from pybossa.exporter.task_csv_export import TaskCsvExporter
from pybossa.exporter.json_export import JsonExporter
from pybossa.exporter.task_json_export import TaskJsonExporter
from pybossa.exporter.project_csv_export import ProjectCsvExporter
csv_exporter = CsvExporter()
task_csv_exporter = TaskCsvExporter()
json_exporter = JsonExporter()
task_json_exporter = TaskJsonExporter()
project_csv_exporter = ProjectCsvExporter()
def export_userdata(user_id, admin_addr, **kwargs):
from pybossa.core import (user_repo, uploader)
from flask import current_app, url_for
json_exporter = JsonExporter()
user = user_repo.get(user_id)
user_data = user.dictize()
del user_data['passwd_hash']
pdf = json_exporter._make_zip(None, '', 'personal_data', user_data, user_id,
'personal_data.zip')
upload_method = current_app.config.get('UPLOAD_METHOD')
attachments = []
personal_data_link = None
if upload_method == 'local':
filename = uploader.get_file_path('user_%s' % user_id, pdf)
with open(filename) as fp:
attachment = Attachment(pdf, "application/zip",
fp.read())
attachments = [attachment]
else:
personal_data_link = url_for(upload_method,
filename="user_%s/%s" % (user_id, pdf))
body = render_template('/account/email/exportdata.md',
user=user.dictize(),
personal_data_link=personal_data_link,
config=current_app.config)
html = render_template('/account/email/exportdata.html',
user=user.dictize(),
personal_data_link=personal_data_link,
config=current_app.config)
subject = 'Your personal data'
bcc = [admin_addr]
mail_dict = dict(recipients=[user.email_addr],
bcc=bcc,
subject=subject,
body=body,
html=html,
attachments=attachments)
send_mail(mail_dict)
def test_exporters_generates_zip(self, json_uploader, csv_uploader,
dataframe):
"""Test that CsvExporter and JsonExporter generate zip works."""
user = UserFactory.create(admin=True)
project = ProjectFactory.create(name='test_project')
expected_gold_answer = {
'best_job': 'software developer',
'best_boss': 'Juan'
}
task1 = TaskFactory.create(project=project,
gold_answers=expected_gold_answer)
task2 = TaskFactory.create(project=project)
task_run = TaskRunFactory.create(project=project, task=task1)
task_run = TaskRunFactory.create(project=project, task=task2)
csv_exporter = CsvExporter()
json_exporter = JsonExporter()
csv_exporter.pregenerate_zip_files(project)
call_dataframe = dataframe.call_args_list
call_csv_params = csv_uploader.upload_file.call_args_list
expected_csv_params = set([
'1_project1_task_run_csv.zip', '1_project1_result_csv.zip',
'1_project1_task_csv.zip'
])
assert self._check_func_called_with_params(call_csv_params,
expected_csv_params)
task1_data = call_dataframe[0][0][0][0]
task2_data = call_dataframe[0][0][0][1]
expected_headers = [
'info', 'fav_user_ids', 'user_pref', 'n_answers', 'quorum',
'calibration', 'created', 'state', 'gold_answers_best_job',
'gold_answers_best_boss', 'exported', 'project_id', 'id',
'priority_0', 'expiration'
]
obj_keys = task1_data.keys()
self._compare_object_keys(obj_keys, expected_headers)
assert task1_data['gold_answers_best_job'] == expected_gold_answer[
'best_job']
assert task1_data['gold_answers_best_boss'] == expected_gold_answer[
'best_boss']
expected_headers = [
'info', 'fav_user_ids', 'user_pref', 'n_answers', 'quorum',
'calibration', 'created', 'state', 'gold_answers', 'exported',
'project_id', 'id', 'priority_0', 'expiration'
]
obj_keys = task2_data.keys()
self._compare_object_keys(obj_keys, expected_headers)
assert task2_data['gold_answers'] == None
json_exporter.pregenerate_zip_files(project)
call_json_params = json_uploader.upload_file.call_args_list
expected_json_params = set([
'1_project1_task_run_json.zip', '1_project1_result_json.zip',
'1_project1_task_json.zip'
])
assert self._check_func_called_with_params(call_json_params,
expected_json_params)
def _export_category_results_as_geoJSON(category_name):
from pybossa.cache import projects as cached_projects
from pybossa.exporter.json_export import JsonExporter
import json
import pandas as pd
import numpy as np
geotagx_json_exporter = JsonExporter()
max_number_of_exportable_projects = 15
projects_in_category = cached_projects.get(category_name, page=1, per_page=max_number_of_exportable_projects)
task_runs = []
task_runs_info = []
project_name_id_mapping = {}
project_id_name_mapping = {}
project_question_type_mapping = {}
project_question_question_text_mapping = {}
for project in projects_in_category:
short_name = project['short_name']
project_id_name_mapping[project['id']] = project['short_name']
project_name_id_mapping[project['short_name']] = project['id']
# Check if it a supported geotagx project whose schema we know
if 'GEOTAGX_SUPPORTED_PROJECTS_SCHEMA' in current_app.config.keys() \
and short_name in current_app.config['GEOTAGX_SUPPORTED_PROJECTS_SCHEMA'].keys():
##Read the project schema and store the respective questions and their types
for _question in current_app.config['GEOTAGX_SUPPORTED_PROJECTS_SCHEMA'][short_name]['questions']:
project_question_type_mapping[unicode(short_name+"::"+_question['answer']['saved_as'])] = _question['type']
project_question_question_text_mapping[unicode(short_name+"::"+_question['answer']['saved_as']+"::question_text")] = _question['title']
#Only export results of known GEOTAGX projects that are created with `geotagx-project-template`
task_runs_generator = geotagx_json_exporter.gen_json("task_run", project['id'])
_task_runs = ""
for task_run_c in task_runs_generator:
_task_runs += task_run_c
task_runs = task_runs + json.loads(_task_runs)
def extract_geotagx_info(json):
"""Returns a list of only info objects of the task_run"""
exploded_json = []
for item in json:
item['info']['project_id'] = item['project_id']
exploded_json.append(item['info'])
return exploded_json
def _summarize_geolocations(geolocation_responses):
"""
TODO :: Add different geo-summarization methods (ConvexHull, Centroid, etc)
"""
responses = []
for response in geolocation_responses:
if type(response) == type([]):
responses.append(response)
return responses
"""
Changes projection to WGS84 projection from WebMercator projection
so that most geojson renderers support it out of the box
Inspired by : http://www.gal-systems.com/2011/07/convert-coordinates-between-web.html
"""
def _project_coordinate_from_webmercator_toWGS84(coordinates):
import math
mercatorX_lon = coordinates[0]
mercatorY_lat = coordinates[1]
if math.fabs(mercatorX_lon) < 180 and math.fabs(mercatorY_lat) < 90:
return False, False
if ((math.fabs(mercatorX_lon) > 20037508.3427892) or (math.fabs(mercatorY_lat) > 20037508.3427892)):
return False, False
x = mercatorX_lon
y = mercatorY_lat
num3 = x / 6378137.0
num4 = num3 * 57.295779513082323
num5 = math.floor(float((num4 + 180.0) / 360.0))
num6 = num4 - (num5 * 360.0)
num7 = 1.5707963267948966 - (2.0 * math.atan(math.exp((-1.0 * y) / 6378137.0)));
mercatorX_lon = num6
mercatorY_lat = num7 * 57.295779513082323
return mercatorX_lon, mercatorY_lat
"""
Changes the projection of the multi_polygon object to WGS84 from WebMercator
"""
def _project_geosummary_from_webmercator_to_WGS84(multi_polygon):
_multi_polygon = []
for polygon in multi_polygon:
_polygon = []
for coordinates in polygon:
try:
_x, _y = _project_coordinate_from_webmercator_toWGS84(coordinates)
if _x and _y:
_polygon.append([_x, _y])
except:
pass # Pass Silentily if there is some error in the input
_multi_polygon.append(_polygon)
return _multi_polygon
def _build_geo_json(geolocation_responses):
geoJSON = {}
geoJSON['type'] = "FeatureCollection"
geoJSON['features'] = []
for response in geolocation_responses:
if response['_geotagx_geolocation_key']:
geo_summary = response[response['_geotagx_geolocation_key']]
_feature = {}
_feature['type'] = "Feature"
_feature['geometry'] = {}
_feature['geometry']['type'] = "MultiPolygon"
_feature['geometry']['coordinates'] = \
[_project_geosummary_from_webmercator_to_WGS84(geo_summary['geo_summary'])]
del response[response['_geotagx_geolocation_key']]
del response['_geotagx_geolocation_key']
_feature['properties'] = response
#Neglect responses with no coordinate labels
if _feature['geometry']['coordinates'] != [[]]:
geoJSON['features'].append(_feature)
return geoJSON
task_runs_info = extract_geotagx_info(task_runs)
task_runs_info = pd.read_json(json.dumps(task_runs_info))
summary_dict = {}
for img_url in task_runs_info['img'].unique():
per_url_data = task_runs_info[task_runs_info['img'] == img_url]
for project_id in np.unique(per_url_data['project_id'].values):
per_summary_dict = {}
per_summary_dict['_geotagx_geolocation_key'] = False
if img_url in summary_dict.keys():
per_summary_dict = summary_dict[img_url]
per_summary_dict['GEOTAGX_IMAGE_URL'] = img_url
per_url_data_project_slice = per_url_data[per_url_data['project_id'] == project_id]
for key in per_url_data_project_slice.keys():
namespaced_key = project_id_name_mapping[project_id]+"::"+key
if key not in ['img', 'isMigrated', 'son_app_id', 'task_id', 'project_id']:
if namespaced_key in project_question_type_mapping.keys():
if project_question_type_mapping[namespaced_key] == u"geotagging":
per_summary_dict['_geotagx_geolocation_key'] = namespaced_key
per_summary_dict[namespaced_key] = {'geo_summary' : _summarize_geolocations(per_url_data_project_slice[key].values)}
else:
per_summary_dict[namespaced_key] = {'answer_summary':dict(per_url_data_project_slice[key].value_counts())}
per_summary_dict[namespaced_key]['question_text'] = project_question_question_text_mapping[unicode(namespaced_key+"::question_text")]
elif key == u"img":
per_summary_dict[project_id_name_mapping[project_id]+"::GEOTAGX_TOTAL"] = len(per_url_data_project_slice)
summary_dict[img_url] = per_summary_dict
geo_json = _build_geo_json(summary_dict.values())
return jsonify(geo_json)
