Esempio n. 1
0
    def test_exporters_generates_zip(self, json_uploader, csv_uploader):
        """Test that CsvExporter and JsonExporter generate zip works."""

        user = UserFactory.create(admin=True)
        project = ProjectFactory.create(name='test_project')
        task = TaskFactory.create(project=project)
        task_run = TaskRunFactory.create(project=project, task=task)
        csv_exporter = CsvExporter()
        json_exporter = JsonExporter()
        csv_exporter.pregenerate_zip_files(project)
        call_csv_params = csv_uploader.upload_file.call_args_list
        expected_csv_params = set(['1_project1_task_run_csv.zip', '1_project1_result_csv.zip', '1_project1_task_csv.zip'])
        assert self._check_func_called_with_params(call_csv_params, expected_csv_params)

        json_exporter.pregenerate_zip_files(project)
        call_json_params = json_uploader.upload_file.call_args_list
        expected_json_params = set(['1_project1_task_run_json.zip', '1_project1_result_json.zip', '1_project1_task_json.zip'])
        assert self._check_func_called_with_params(call_json_params, expected_json_params)
Esempio n. 2
0
def export_userdata(user_id, **kwargs):
    from pybossa.core import user_repo, project_repo, task_repo, result_repo
    from flask import current_app, url_for
    json_exporter = JsonExporter()
    user = user_repo.get(user_id)
    user_data = user.dictize()
    del user_data['passwd_hash']
    projects = project_repo.filter_by(owner_id=user.id)
    projects_data = [project.dictize() for project in projects]
    taskruns = task_repo.filter_task_runs_by(user_id=user.id)
    taskruns_data = [tr.dictize() for tr in taskruns]
    pdf = json_exporter._make_zip(None, '', 'personal_data', user_data, user_id,
                                  'personal_data.zip')
    upf = json_exporter._make_zip(None, '', 'user_projects', projects_data, user_id,
                                  'user_projects.zip')
    ucf = json_exporter._make_zip(None, '', 'user_contributions', taskruns_data, user_id,
                                  'user_contributions.zip')
    upload_method = current_app.config.get('UPLOAD_METHOD')
    if upload_method == 'local':
        upload_method = 'uploads.uploaded_file'

    personal_data_link = url_for(upload_method,
                                 filename="user_%s/%s" % (user_id, pdf))
    personal_projects_link = url_for(upload_method,
                                    filename="user_%s/%s" % (user_id,
                                                             upf))
    personal_contributions_link = url_for(upload_method,
                                          filename="user_%s/%s" % (user_id,
                                                                   ucf))

    body = render_template('/account/email/exportdata.md',
                           user=user.dictize(),
                           personal_data_link=personal_data_link,
                           personal_projects_link=personal_projects_link,
                           personal_contributions_link=personal_contributions_link,
                           config=current_app.config)

    html = render_template('/account/email/exportdata.html',
                           user=user.dictize(),
                           personal_data_link=personal_data_link,
                           personal_projects_link=personal_projects_link,
                           personal_contributions_link=personal_contributions_link,
                           config=current_app.config)
    subject = 'Your personal data'
    mail_dict = dict(recipients=[user.email_addr],
                     subject=subject,
                     body=body,
                     html=html)
    send_mail(mail_dict)
Esempio n. 3
0
def setup_exporter(app):
    """Setup exporter."""
    global csv_exporter
    global task_csv_exporter
    global json_exporter
    global task_json_exporter
    global project_csv_exporter
    from pybossa.exporter.csv_export import CsvExporter
    from pybossa.exporter.task_csv_export import TaskCsvExporter
    from pybossa.exporter.json_export import JsonExporter
    from pybossa.exporter.task_json_export import TaskJsonExporter
    from pybossa.exporter.project_csv_export import ProjectCsvExporter
    csv_exporter = CsvExporter()
    task_csv_exporter = TaskCsvExporter()
    json_exporter = JsonExporter()
    task_json_exporter = TaskJsonExporter()
    project_csv_exporter = ProjectCsvExporter()
Esempio n. 4
0
def export_userdata(user_id, admin_addr, **kwargs):
    from pybossa.core import (user_repo, uploader)
    from flask import current_app, url_for
    json_exporter = JsonExporter()
    user = user_repo.get(user_id)
    user_data = user.dictize()
    del user_data['passwd_hash']
    pdf = json_exporter._make_zip(None, '', 'personal_data', user_data, user_id,
                                  'personal_data.zip')
    upload_method = current_app.config.get('UPLOAD_METHOD')

    attachments = []
    personal_data_link = None
    if upload_method == 'local':
        filename = uploader.get_file_path('user_%s' % user_id, pdf)
        with open(filename) as fp:
            attachment = Attachment(pdf, "application/zip",
                                    fp.read())
        attachments = [attachment]
    else:
        personal_data_link = url_for(upload_method,
                                    filename="user_%s/%s" % (user_id, pdf))

    body = render_template('/account/email/exportdata.md',
                           user=user.dictize(),
                           personal_data_link=personal_data_link,
                           config=current_app.config)

    html = render_template('/account/email/exportdata.html',
                           user=user.dictize(),
                           personal_data_link=personal_data_link,
                           config=current_app.config)
    subject = 'Your personal data'
    bcc = [admin_addr]
    mail_dict = dict(recipients=[user.email_addr],
                     bcc=bcc,
                     subject=subject,
                     body=body,
                     html=html,
                     attachments=attachments)
    send_mail(mail_dict)
    def test_exporters_generates_zip(self, json_uploader, csv_uploader,
                                     dataframe):
        """Test that CsvExporter and JsonExporter generate zip works."""
        user = UserFactory.create(admin=True)
        project = ProjectFactory.create(name='test_project')
        expected_gold_answer = {
            'best_job': 'software developer',
            'best_boss': 'Juan'
        }

        task1 = TaskFactory.create(project=project,
                                   gold_answers=expected_gold_answer)
        task2 = TaskFactory.create(project=project)

        task_run = TaskRunFactory.create(project=project, task=task1)
        task_run = TaskRunFactory.create(project=project, task=task2)

        csv_exporter = CsvExporter()
        json_exporter = JsonExporter()
        csv_exporter.pregenerate_zip_files(project)
        call_dataframe = dataframe.call_args_list
        call_csv_params = csv_uploader.upload_file.call_args_list
        expected_csv_params = set([
            '1_project1_task_run_csv.zip', '1_project1_result_csv.zip',
            '1_project1_task_csv.zip'
        ])
        assert self._check_func_called_with_params(call_csv_params,
                                                   expected_csv_params)

        task1_data = call_dataframe[0][0][0][0]
        task2_data = call_dataframe[0][0][0][1]

        expected_headers = [
            'info', 'fav_user_ids', 'user_pref', 'n_answers', 'quorum',
            'calibration', 'created', 'state', 'gold_answers_best_job',
            'gold_answers_best_boss', 'exported', 'project_id', 'id',
            'priority_0', 'expiration'
        ]
        obj_keys = task1_data.keys()

        self._compare_object_keys(obj_keys, expected_headers)
        assert task1_data['gold_answers_best_job'] == expected_gold_answer[
            'best_job']
        assert task1_data['gold_answers_best_boss'] == expected_gold_answer[
            'best_boss']

        expected_headers = [
            'info', 'fav_user_ids', 'user_pref', 'n_answers', 'quorum',
            'calibration', 'created', 'state', 'gold_answers', 'exported',
            'project_id', 'id', 'priority_0', 'expiration'
        ]
        obj_keys = task2_data.keys()

        self._compare_object_keys(obj_keys, expected_headers)
        assert task2_data['gold_answers'] == None

        json_exporter.pregenerate_zip_files(project)
        call_json_params = json_uploader.upload_file.call_args_list
        expected_json_params = set([
            '1_project1_task_run_json.zip', '1_project1_result_json.zip',
            '1_project1_task_json.zip'
        ])
        assert self._check_func_called_with_params(call_json_params,
                                                   expected_json_params)
Esempio n. 6
0
def _export_category_results_as_geoJSON(category_name):
    from pybossa.cache import projects as cached_projects
    from pybossa.exporter.json_export import JsonExporter
    import json
    import pandas as pd
    import numpy as np

    geotagx_json_exporter = JsonExporter()

    max_number_of_exportable_projects = 15
    projects_in_category = cached_projects.get(category_name, page=1, per_page=max_number_of_exportable_projects)
    task_runs = []
    task_runs_info = []
    project_name_id_mapping = {}
    project_id_name_mapping = {}

    project_question_type_mapping = {}
    project_question_question_text_mapping = {}

    for project in projects_in_category:
        short_name = project['short_name']

        project_id_name_mapping[project['id']] = project['short_name']
        project_name_id_mapping[project['short_name']] = project['id']

        # Check if it a supported geotagx project whose schema we know
        if 'GEOTAGX_SUPPORTED_PROJECTS_SCHEMA' in current_app.config.keys() \
            and short_name in current_app.config['GEOTAGX_SUPPORTED_PROJECTS_SCHEMA'].keys():

            ##Read the project schema and store the respective questions and their types
            for _question in current_app.config['GEOTAGX_SUPPORTED_PROJECTS_SCHEMA'][short_name]['questions']:
                project_question_type_mapping[unicode(short_name+"::"+_question['answer']['saved_as'])] = _question['type']
                project_question_question_text_mapping[unicode(short_name+"::"+_question['answer']['saved_as']+"::question_text")] = _question['title']

            #Only export results of known GEOTAGX projects that are created with `geotagx-project-template`
            task_runs_generator = geotagx_json_exporter.gen_json("task_run", project['id'])
            _task_runs = ""
            for task_run_c in task_runs_generator:
                _task_runs += task_run_c

            task_runs = task_runs + json.loads(_task_runs)

    def extract_geotagx_info(json):
        """Returns a list of only info objects of the task_run"""
        exploded_json = []
        for item in json:
            item['info']['project_id'] = item['project_id']
            exploded_json.append(item['info'])
        return exploded_json

    def _summarize_geolocations(geolocation_responses):
        """
            TODO :: Add different geo-summarization methods (ConvexHull, Centroid, etc)
        """
        responses = []

        for response in geolocation_responses:
            if type(response) == type([]):
                responses.append(response)

        return responses

    """
        Changes projection to WGS84 projection  from WebMercator projection
        so that most geojson renderers support it out of the box
        Inspired by : http://www.gal-systems.com/2011/07/convert-coordinates-between-web.html
    """
    def _project_coordinate_from_webmercator_toWGS84(coordinates):
        import math

        mercatorX_lon = coordinates[0]
        mercatorY_lat = coordinates[1]

        if math.fabs(mercatorX_lon) < 180 and math.fabs(mercatorY_lat) < 90:
            return False, False

        if ((math.fabs(mercatorX_lon) > 20037508.3427892) or (math.fabs(mercatorY_lat) > 20037508.3427892)):
            return False, False

        x = mercatorX_lon
        y = mercatorY_lat
        num3 = x / 6378137.0
        num4 = num3 * 57.295779513082323
        num5 = math.floor(float((num4 + 180.0) / 360.0))
        num6 = num4 - (num5 * 360.0)
        num7 = 1.5707963267948966 - (2.0 * math.atan(math.exp((-1.0 * y) / 6378137.0)));
        mercatorX_lon = num6
        mercatorY_lat = num7 * 57.295779513082323

        return mercatorX_lon, mercatorY_lat

    """
        Changes the projection of the multi_polygon object to WGS84 from WebMercator
    """
    def _project_geosummary_from_webmercator_to_WGS84(multi_polygon):
        _multi_polygon = []
        for polygon in multi_polygon:
            _polygon = []
            for coordinates in polygon:
                try:
                    _x, _y = _project_coordinate_from_webmercator_toWGS84(coordinates)
                    if _x and _y:
                        _polygon.append([_x, _y])
                except:
                    pass # Pass Silentily if there is some error in the input
            _multi_polygon.append(_polygon)
        return _multi_polygon

    def _build_geo_json(geolocation_responses):
        geoJSON = {}
        geoJSON['type'] = "FeatureCollection"
        geoJSON['features'] = []
        for response in geolocation_responses:
            if response['_geotagx_geolocation_key']:
                geo_summary = response[response['_geotagx_geolocation_key']]
                _feature = {}
                _feature['type'] = "Feature"
                _feature['geometry'] = {}

                _feature['geometry']['type'] = "MultiPolygon"
                _feature['geometry']['coordinates'] = \
                    [_project_geosummary_from_webmercator_to_WGS84(geo_summary['geo_summary'])]

                del response[response['_geotagx_geolocation_key']]
                del response['_geotagx_geolocation_key']
                _feature['properties'] = response

                #Neglect responses with no coordinate labels
                if _feature['geometry']['coordinates'] != [[]]:
                    geoJSON['features'].append(_feature)

        return geoJSON

    task_runs_info = extract_geotagx_info(task_runs)
    task_runs_info = pd.read_json(json.dumps(task_runs_info))

    summary_dict = {}
    for img_url in task_runs_info['img'].unique():
        per_url_data = task_runs_info[task_runs_info['img'] == img_url]

        for project_id in np.unique(per_url_data['project_id'].values):

            per_summary_dict = {}
            per_summary_dict['_geotagx_geolocation_key'] = False

            if img_url in summary_dict.keys():
                per_summary_dict = summary_dict[img_url]

            per_summary_dict['GEOTAGX_IMAGE_URL'] = img_url
            per_url_data_project_slice = per_url_data[per_url_data['project_id'] == project_id]

            for key in per_url_data_project_slice.keys():
                namespaced_key = project_id_name_mapping[project_id]+"::"+key
                if key not in ['img', 'isMigrated', 'son_app_id', 'task_id', 'project_id']:
                    if namespaced_key in project_question_type_mapping.keys():
                        if project_question_type_mapping[namespaced_key] == u"geotagging":
                            per_summary_dict['_geotagx_geolocation_key'] = namespaced_key
                            per_summary_dict[namespaced_key] = {'geo_summary' : _summarize_geolocations(per_url_data_project_slice[key].values)}
                        else:
                            per_summary_dict[namespaced_key] = {'answer_summary':dict(per_url_data_project_slice[key].value_counts())}
                        per_summary_dict[namespaced_key]['question_text'] = project_question_question_text_mapping[unicode(namespaced_key+"::question_text")]

                elif key == u"img":
                    per_summary_dict[project_id_name_mapping[project_id]+"::GEOTAGX_TOTAL"] = len(per_url_data_project_slice)

            summary_dict[img_url] = per_summary_dict

    geo_json = _build_geo_json(summary_dict.values())
    return jsonify(geo_json)