예제 #1
0
def location_headers(pk):
    user = current_user._get_current_object()

    # disallow processing other users' files
    upload = services.user_uploads.get_or_404(pk=pk, user=user)
    try:
        dataframe = load_source_file(upload.data)
    except Exception:
        # delete loaded file
        upload.data.delete()
        upload.delete()
        return abort(400)

    deployment = g.deployment
    headers = dataframe.columns
    template_name = 'frontend/location_headers.html'

    if request.method == 'GET':
        form = generate_location_update_mapping_form(deployment, headers)
        return render_template(template_name, form=form)
    else:
        form = generate_location_update_mapping_form(deployment, headers,
                                                     request.form)

        if not form.validate():
            return render_template(template_name, form=form)
        else:
            # get header mappings
            data = form.data.copy()

            # invoke task asynchronously
            kwargs = {'upload_id': unicode(upload.id), 'mappings': data}
            tasks.import_locations.apply_async(kwargs=kwargs)

            return redirect(url_for('locations.locations_list'))
예제 #2
0
def import_locations(self, upload_id, mappings, location_set_id, channel=None):
    upload = UserUpload.query.filter(UserUpload.id == upload_id).first()
    if not upload:
        logger.error('Upload %s does not exist, aborting', upload_id)
        return

    filepath = uploads.path(upload.upload_filename)

    if not os.path.exists(filepath):
        logger.error('Upload file %s does not exist, aborting', filepath)
        upload.delete()
        return

    with open(filepath, 'rb') as f:
        dataframe = helpers.load_source_file(f)

    location_set = LocationSet.query.filter(
        LocationSet.id == location_set_id).first()

    update_locations(dataframe, mappings, location_set, self)

    os.remove(filepath)
    upload.delete()

    msg_body = render_template_string(email_template)

    send_email(_('Locations Import Report'), msg_body, [upload.user.email])
예제 #3
0
def make_import_mapping_form(import_file, location_set):
    attributes = {'location_set': location_set, 'validate': _validate_mappings}
    map_choices = _make_choices(location_set.get_import_fields().items())

    data_frame = load_source_file(import_file)
    for index, column in enumerate(data_frame.columns):
        attributes[str(index)] = SelectField(column, choices=map_choices)

    return type('LocationImportMapForm', (FlaskForm, ), attributes)
예제 #4
0
def make_import_mapping_form(import_file, participant_set):
    attributes = {'validate': _validate_required_fields}
    map_choices = _make_choices(participant_set.get_import_fields().items())

    data_frame = load_source_file(import_file)
    for column in data_frame.columns:
        attributes[slugify(column)] = SelectField(column, choices=map_choices)

    return type('ParticipantImportMapForm', (FlaskForm, ), attributes)
예제 #5
0
def import_participants(upload_id, mappings):
    upload = services.user_uploads.get(pk=upload_id)
    dataframe = helpers.load_source_file(upload.data)
    count, errors, warnings = update_participants(dataframe, upload.event,
                                                  mappings)

    # delete uploaded file
    upload.data.delete()
    upload.delete()

    msg_body = generate_response_email(count, errors, warnings)

    send_email(_('Import report'), msg_body, [upload.user.email])
예제 #6
0
파일: tasks.py 프로젝트: nditech/elections
def import_participants(upload_id, mappings):
    upload = services.user_uploads.get(pk=upload_id)
    dataframe = helpers.load_source_file(upload.data)
    count, errors, warnings = update_participants(
        dataframe,
        upload.event,
        mappings
    )

    # delete uploaded file
    upload.data.delete()
    upload.delete()

    msg_body = generate_response_email(count, errors, warnings)

    send_email(_('Import report'), msg_body, [upload.user.email])
예제 #7
0
def participant_headers(pk):
    user = current_user._get_current_object()

    # disallow processing other users' files
    upload = services.user_uploads.get_or_404(pk=pk, user=user)
    try:
        dataframe = load_source_file(upload.data)
    except Exception:
        # delete loaded file
        upload.data.delete()
        upload.delete()
        return abort(400)

    deployment = g.deployment
    headers = dataframe.columns
    template_name = 'frontend/participant_headers.html'

    if request.method == 'GET':
        form = generate_participant_import_mapping_form(deployment, headers)
        return render_template(template_name, form=form)
    else:
        form = generate_participant_import_mapping_form(
            deployment, headers, request.form)

        if not form.validate():
            return render_template(
                template_name,
                form=form
            )
        else:
            # get header mappings
            data = form.data.copy()

            # phone_header = data.pop('phone')
            # group_header = data.pop('group')
            # mappings = {v: k for k, v in data.iteritems() if v}
            # mappings.update(phone=phone_header)
            # mappings.update(group=group_header)

            # invoke task asynchronously
            kwargs = {
                'upload_id': unicode(upload.id),
                'mappings': data
            }
            import_participants.apply_async(kwargs=kwargs)

            return redirect(url_for('participants.participant_list'))
예제 #8
0
def location_headers(pk):
    user = current_user._get_current_object()

    # disallow processing other users' files
    upload = services.user_uploads.get_or_404(pk=pk, user=user)
    try:
        dataframe = load_source_file(upload.data)
    except Exception:
        # delete loaded file
        upload.data.delete()
        upload.delete()
        return abort(400)

    deployment = g.deployment
    headers = dataframe.columns
    template_name = 'frontend/location_headers.html'

    if request.method == 'GET':
        form = generate_location_update_mapping_form(deployment, headers)
        return render_template(template_name, form=form)
    else:
        form = generate_location_update_mapping_form(
            deployment, headers, request.form)

        if not form.validate():
            return render_template(
                template_name,
                form=form
            )
        else:
            # get header mappings
            data = form.data.copy()

            # invoke task asynchronously
            kwargs = {
                'upload_id': unicode(upload.id),
                'mappings': data
            }
            import_locations.apply_async(kwargs=kwargs)

            return redirect(url_for('locations.locations_list'))
예제 #9
0
def participant_headers(pk):
    user = current_user._get_current_object()

    # disallow processing other users' files
    upload = services.user_uploads.get_or_404(pk=pk, user=user)
    try:
        dataframe = load_source_file(upload.data)
    except Exception:
        # delete loaded file
        upload.data.delete()
        upload.delete()
        return abort(400)

    deployment = g.deployment
    headers = dataframe.columns
    template_name = 'frontend/participant_headers.html'

    if request.method == 'GET':
        form = generate_participant_import_mapping_form(deployment, headers)
        return render_template(template_name, form=form)
    else:
        form = generate_participant_import_mapping_form(
            deployment, headers, request.form)

        if not form.validate():
            return render_template(template_name, form=form)
        else:
            # get header mappings
            data = form.data.copy()

            # phone_header = data.pop('phone')
            # group_header = data.pop('group')
            # mappings = {v: k for k, v in data.iteritems() if v}
            # mappings.update(phone=phone_header)
            # mappings.update(group=group_header)

            # invoke task asynchronously
            kwargs = {'upload_id': unicode(upload.id), 'mappings': data}
            import_participants.apply_async(kwargs=kwargs)

            return redirect(url_for('participants.participant_list'))
예제 #10
0
파일: tasks.py 프로젝트: nditech/elections
def import_locations(upload_id, mappings):
    upload = services.user_uploads.get(pk=upload_id)
    dataframe = helpers.load_source_file(upload.data)

    update_locations(dataframe, mappings, upload.event)

    # fetch and update all submissions
    deployment = upload.event.deployment
    submissions = services.submissions.all().filter(deployment=deployment)
    for submission in submissions:
        if submission.location:
            submission.location_name_path = helpers.compute_location_path(submission.location)
            submission.save(clean=False)

    # delete uploaded file
    upload.data.delete()
    upload.delete()

    msg_body = render_template_string(email_template)

    send_email(_("Locations Import Report"), msg_body, [upload.user.email])
예제 #11
0
def import_locations(upload_id, mappings):
    upload = services.user_uploads.get(pk=upload_id)
    dataframe = helpers.load_source_file(upload.data)

    update_locations(dataframe, mappings, upload.event)

    # fetch and update all submissions
    deployment = upload.event.deployment
    submissions = services.submissions.all().filter(deployment=deployment)
    for submission in submissions:
        if submission.location:
            submission.location_name_path = helpers.compute_location_path(
                submission.location)
            submission.save(clean=False)

    # delete uploaded file
    upload.data.delete()
    upload.delete()

    msg_body = render_template_string(email_template)

    send_email(_('Locations Import Report'), msg_body, [upload.user.email])
예제 #12
0
def import_participants(
        self, upload_id, mappings, participant_set_id, channel=None):
    upload = services.user_uploads.find(id=upload_id).first()
    if not upload:
        logger.error('Upload %s does not exist, aborting', upload_id)
        return

    filepath = uploads.path(upload.upload_filename)
    if not os.path.exists(filepath):
        logger.error('Upload file %s does not exist, aborting', filepath)
        upload.delete()
        return

    with open(filepath, 'rb') as f:
        dataframe = helpers.load_source_file(f)

    participant_set = services.participant_sets.find(
        id=participant_set_id).first()

    if not participant_set:
        _cleanup_upload(filepath, upload)
        logger.error(
            'Participant set with id {} does not exist, aborting'.format(
                participant_set_id))

    count, errors, warnings = update_participants(
        dataframe,
        mappings,
        participant_set,
        self
    )

    # delete uploaded file
    os.remove(filepath)
    upload.delete()

    msg_body = generate_response_email(count, errors, warnings)

    send_email(_('Import report'), msg_body, [upload.user.email])
예제 #13
0
def init_survey_submissions(self, event_id, form_id, upload_id):
    event = models.Event.query.filter_by(id=event_id).first()
    form = models.Form.query.filter_by(id=form_id).first()

    upload = UserUpload.query.filter(UserUpload.id == upload_id).first()
    if not upload:
        logger.error('Upload %s does not exist, aborting', upload_id)
        return

    filepath = uploads.path(upload.upload_filename)

    if not os.path.exists(filepath):
        logger.error('Upload file %s does not exist, aborting', filepath)
        upload.delete()
        return

    with open(filepath, 'rb') as f:
        dataframe = helpers.load_source_file(f)

        total_records = dataframe.shape[0]
        processed_records = 0
        error_records = 0
        warning_records = 0
        error_log = []

        for idx in range(len(dataframe.index)):
            row = dataframe.iloc[idx]
            pid = row.get('PARTICIPANT_ID')
            serial = row.get('FORM_SERIAL')

            participant = Participant.query.filter(
                Participant.participant_id == pid,
                Participant.participant_set == event.participant_set).first()

            if not participant:
                error_records += 1
                error_log.append({
                    'label':
                    'ERROR',
                    'message':
                    gettext('Participant ID %(part_id)s was not found',
                            part_id=pid)
                })

                self.update_task_info(total_records=total_records,
                                      processed_records=processed_records,
                                      error_records=error_records,
                                      warning_records=warning_records,
                                      error_log=error_log)

                continue

            if not serial:
                error_records += 1
                error_log.append({
                    'label':
                    'ERROR',
                    'message':
                    gettext(
                        'No form serial number specified on line %(lineno)d',
                        lineno=idx + 1)
                })

                self.update_task_info(total_records=total_records,
                                      processed_records=processed_records,
                                      error_records=error_records,
                                      warning_records=warning_records,
                                      error_log=error_log)

                continue

            submission = Submission.query.filter(
                Submission.form == form, Submission.participant == participant,
                Submission.location == participant.location,
                Submission.deployment == event.deployment,
                Submission.event == event, Submission.serial_no == serial,
                Submission.submission_type == 'O').first()

            if not submission:
                submission = Submission(form_id=form.id,
                                        participant_id=participant.id,
                                        location_id=participant.location.id,
                                        deployment_id=event.deployment.id,
                                        event_id=event.id,
                                        submission_type='O',
                                        serial_no=serial,
                                        data={})
                submission.save()

            master = Submission.query.filter(
                Submission.form == form,
                Submission.participant_id == None,  # noqa
                Submission.location == participant.location,
                Submission.deployment == event.deployment,
                Submission.event == event,
                Submission.serial_no == serial,
                Submission.submission_type == 'M').first()

            if not master:
                master = Submission(form_id=form.id,
                                    participant_id=None,
                                    location_id=participant.location.id,
                                    deployment_id=event.deployment.id,
                                    event_id=event.id,
                                    submission_type='M',
                                    serial_no=serial,
                                    data={})
                master.save()

            processed_records += 1

            self.update_task_info(total_records=total_records,
                                  processed_records=processed_records,
                                  error_records=error_records,
                                  warning_records=warning_records,
                                  error_log=error_log)