Example #1
0
    def clean_contact_upload_file(self):
        value = self.cleaned_data.get("contact_upload_file", None)
        if self.cleaned_data.get("use_contact_upload_file", "N") == "Y":
            if value is None:
                raise ValidationError("Please choose a file.")

            try:
                workbook = WorkbookJSONReader(value)
            except InvalidFileException:
                raise ValidationError(
                    "Invalid format. Please convert to Excel 2007 or higher (.xlsx) and try again."
                )

            try:
                worksheet = workbook.get_worksheet()
            except WorksheetNotFound:
                raise ValidationError("Workbook has no worksheets.")

            contacts = []
            for row in worksheet:
                if "PhoneNumber" not in row:
                    raise ValidationError("Column 'PhoneNumber' not found.")
                contacts.append({
                    "phone_number":
                    validate_phone_number(row.get("PhoneNumber"))
                })

            if len(contacts) == 0:
                raise ValidationError(_("Please add at least one contact."))

            return contacts
        else:
            return None
Example #2
0
def update_domains(request):
    if request.method == "POST":
        try:
            workbook = WorkbookJSONReader(request.file)
            domains = workbook.get_worksheet(title='domains')
            success_count = 0
            fail_count = 0
            for row in domains:
                try:
                    name = row["name"]
                    domain = Domain.get_by_name(name)
                    if domain:
                        for k, v in row.items():
                            setattr(domain, k, v)
                        domain.save()
                        success_count += 1
                    else:
                        messages.warning(request, "No domain with name %s found" % name)
                        fail_count += 1
                except Exception, e:
                    messages.warning(request, "Update for %s failed: %s" % (row.get("name", '<No Name>'), e))
                    fail_count += 1
            if success_count:
                messages.success(request, "%s domains successfully updated" % success_count)
            if fail_count:
                messages.error(request, "%s domains had errors. details above." % fail_count)
            
        except Exception, e:
            messages.error(request, "Something went wrong! Update failed. Here's your error: %s" % e)
Example #3
0
 def clean_contact_upload_file(self):
     value = self.cleaned_data.get("contact_upload_file", None)
     if self.cleaned_data.get("use_contact_upload_file", "N") == "Y":
         if value is None:
             raise ValidationError("Please choose a file.")
         
         try:
             workbook = WorkbookJSONReader(value)
         except InvalidFileException:
             raise ValidationError("Invalid format. Please convert to Excel 2007 or higher (.xlsx) and try again.")
         
         try:
             worksheet = workbook.get_worksheet()
         except WorksheetNotFound:
             raise ValidationError("Workbook has no worksheets.")
         
         contacts = []
         for row in worksheet:
             if "PhoneNumber" not in row:
                 raise ValidationError("Column 'PhoneNumber' not found.")
             contacts.append({"phone_number" : validate_phone_number(row.get("PhoneNumber"))})
         
         if len(contacts) == 0:
             raise ValidationError(_("Please add at least one contact."))
         
         return contacts
     else:
         return None
Example #4
0
def update_domains(request):
    if request.method == "POST":
        try:
            workbook = WorkbookJSONReader(request.file)
            domains = workbook.get_worksheet(title="domains")
            success_count = 0
            fail_count = 0
            for row in domains:
                try:
                    name = row["name"]
                    domain = Domain.get_by_name(name)
                    if domain:
                        for k, v in row.items():
                            setattr(domain, k, v)
                        domain.save()
                        success_count += 1
                    else:
                        messages.warning(request, "No domain with name %s found" % name)
                        fail_count += 1
                except Exception, e:
                    messages.warning(request, "Update for %s failed: %s" % (row.get("name", "<No Name>"), e))
                    fail_count += 1
            if success_count:
                messages.success(request, "%s domains successfully updated" % success_count)
            if fail_count:
                messages.error(request, "%s domains had errors. details above." % fail_count)

        except Exception, e:
            messages.error(request, "Something went wrong! Update failed. Here's your error: %s" % e)
Example #5
0
def upload_sms_translations(request, domain):
    try:
        workbook = WorkbookJSONReader(request.file)
        translations = workbook.get_worksheet(title='translations')

        with StandaloneTranslationDoc.get_locked_obj(domain, "sms") as tdoc:
            msg_ids = sorted(_MESSAGES.keys())
            result = {}
            for lang in tdoc.langs:
                result[lang] = {}

            for row in translations:
                for lang in tdoc.langs:
                    if row.get(lang):
                        msg_id = row["property"]
                        if msg_id in msg_ids:
                            result[lang][msg_id] = str(row[lang]).strip()

            tdoc.translations = result
            tdoc.save()
        messages.success(request, _("SMS Translations Updated."))
    except Exception:
        notify_exception(request, 'SMS Upload Translations Error')
        messages.error(request, _("Update failed. We're looking into it."))

    return HttpResponseRedirect(reverse('sms_languages', args=[domain]))
Example #6
0
    def post(self, request, *args, **kwargs):
        upload = request.FILES.get('bulk_upload_file')
        """View's dispatch method automatically calls this"""
        try:
            self.workbook = WorkbookJSONReader(upload)
        except InvalidFileException:
            try:
                csv.DictReader(
                    io.StringIO(upload.read().decode('ascii'), newline=None))
                return HttpResponseBadRequest(
                    "CommCare HQ no longer supports CSV upload. "
                    "Please convert to Excel 2007 or higher (.xlsx) "
                    "and try again.")
            except UnicodeDecodeError:
                return HttpResponseBadRequest("Unrecognized format")
        except JSONReaderError as e:
            messages.error(request,
                           'Your upload was unsuccessful. %s' % e.message)
            return self.get(request, *args, **kwargs)
        except HeaderValueError as e:
            return HttpResponseBadRequest(
                "Upload encountered a data type error: %s" % e.message)

        try:
            self.user_specs = self.workbook.get_worksheet(title='users')
        except WorksheetNotFound:
            try:
                self.user_specs = self.workbook.get_worksheet()
            except WorksheetNotFound:
                return HttpResponseBadRequest("Workbook has no worksheets")

        try:
            self.group_specs = self.workbook.get_worksheet(title='groups')
        except WorksheetNotFound:
            self.group_specs = []

        self.location_specs = []
        if Domain.get_by_name(self.domain).commtrack_enabled:
            try:
                self.location_specs = self.workbook.get_worksheet(
                    title='locations')
            except WorksheetNotFound:
                # if there is no sheet for locations (since this was added
                # later and is optional) we don't error
                pass

        try:
            check_headers(self.user_specs)
        except UserUploadError as e:
            return HttpResponseBadRequest(e)

        task_ref = expose_download(None, expiry=1 * 60 * 60)
        task = bulk_upload_async.delay(self.domain, list(self.user_specs),
                                       list(self.group_specs),
                                       list(self.location_specs))
        task_ref.set_task(task)
        return HttpResponseRedirect(
            reverse(UserUploadStatusView.urlname,
                    args=[self.domain, task_ref.download_id]))
Example #7
0
 def __init__(self, file_or_filename):
     try:
         self.workbook = WorkbookJSONReader(file_or_filename)
     except AttributeError:
         raise FixtureUploadError(
             _("Error processing your Excel (.xlsx) file"))
     except Exception:
         raise FixtureUploadError(
             _("Invalid file-format. Please upload a valid xlsx file."))
Example #8
0
    def test_archive_forms_basic(self):
        uploaded_file = WorkbookJSONReader(join(BASE_PATH, BASIC_XLSX))

        response = archive_forms(self.domain_name, self.user, list(uploaded_file.get_worksheet()))

        # Need to re-get instance from DB to get updated attributes
        for key, _id in self.XFORMS.iteritems():
            self.assertEqual(XFormInstance.get(_id).doc_type, 'XFormArchived')

        self.assertEqual(len(response['success']), len(self.xforms))
Example #9
0
    def test_archive_forms_missing(self):
        uploaded_file = WorkbookJSONReader(join(BASE_PATH, MISSING_XLSX))

        response = archive_forms(self.domain_name, self.user, list(uploaded_file.get_worksheet()))

        for key, _id in self.XFORMS.iteritems():
            self.assertEqual(XFormInstance.get(_id).doc_type, 'XFormArchived')

        self.assertEqual(len(response['success']), len(self.xforms))
        self.assertEqual(len(response['errors']), 1,
                         "One error for trying to archive a missing form")
Example #10
0
 def clean_mach_file(self):
     if 'mach_file' in self.cleaned_data:
         mach_file = self.cleaned_data['mach_file']
         try:
             mach_file = WorkbookJSONReader(mach_file)
             mach_file = mach_file.get_worksheet()
         except InvalidFileException:
             raise forms.ValidationError("Please convert to Excel 2007 or higher (.xlsx) and try again.")
         except Exception as e:
             raise forms.ValidationError("Encountered error: %s" % e)
         return mach_file
Example #11
0
    def post(self, request):
        """View's dispatch method automatically calls this"""

        try:
            workbook = WorkbookJSONReader(request.file)
        except AttributeError:
            return HttpResponseBadRequest("Error processing your Excel (.xlsx) file")

        try:
            data_types = workbook.get_worksheet(title='types')
        except KeyError:
            return HttpResponseBadRequest("Workbook does not have a sheet called 'types'")

        for dt in data_types:
            print "DataType"
            print dt

            for di in workbook.get_worksheet(title=dt['tag']):
                print "DataItem"
                print di

        for dt in data_types:
            data_type = FixtureDataType(
                domain=self.domain,
                name=dt['name'],
                tag=dt['tag'],
                fields=dt['field'],
            )
            data_type.save()
            data_items = workbook.get_worksheet(data_type.tag)
            for di in data_items:
                print di
                data_item = FixtureDataItem(
                    domain=self.domain,
                    data_type_id=data_type.get_id,
                    fields=di['field']
                )
                data_item.save()
                for group_name in di.get('group', []):
                    group = Group.by_name(self.domain, group_name)
                    if group:
                        data_item.add_group(group)
                    else:
                        messages.error(request, "Unknown group: %s" % group_name)
                for raw_username in di.get('user', []):
                    username = normalize_username(raw_username, self.domain)
                    user = CommCareUser.get_by_username(username)
                    if user:
                        data_item.add_user(user)
                    else:
                        messages.error(request, "Unknown user: %s" % raw_username)

        return HttpResponseRedirect(reverse('fixture_view', args=[self.domain]))
Example #12
0
def import_locations_async(domain, file_ref_id, update_existing=False):
    task = import_locations_async

    DownloadBase.set_progress(task, 0, 100)
    download_ref = DownloadBase.get(file_ref_id)
    workbook = WorkbookJSONReader(download_ref.get_filename())
    worksheet = workbook.get_worksheet()

    results = list(import_locations(domain, worksheet, update_existing, task))

    DownloadBase.set_progress(task, 100, 100)

    return {
        'messages': results
    }
Example #13
0
    def post(self, request):
        """View's dispatch method automatically calls this"""

        def error_redirect():
            return HttpResponseRedirect(reverse('upload_item_lists', args=[self.domain]))

        try:
            workbook = WorkbookJSONReader(request.file)
        except AttributeError:
            messages.error(request, "Error processing your Excel (.xlsx) file")
            return error_redirect()



        try:
            self._run_upload(request, workbook)
        except WorksheetNotFound as e:
            messages.error(request, "Workbook does not have a sheet called '%s'" % e.title)
            return error_redirect()
        except Exception as e:
            notify_exception(request)
            messages.error(request, "Fixture upload could not complete due to the following error: %s" % e)
            return error_redirect()

        return HttpResponseRedirect(reverse('fixture_view', args=[self.domain]))
Example #14
0
 def __init__(self, file_or_filename):
     try:
         self.workbook = WorkbookJSONReader(file_or_filename)
     except AttributeError:
         raise FixtureUploadError(_("Error processing your Excel (.xlsx) file"))
     except Exception:
         raise FixtureUploadError(_("Invalid file-format. Please upload a valid xlsx file."))
Example #15
0
    def post(self, request):
        """View's dispatch method automatically calls this"""

        try:
            self.workbook = WorkbookJSONReader(request.file)
        except InvalidFileException:
            try:
                csv.DictReader(io.StringIO(request.file.read().decode('ascii'), newline=None))
                return HttpResponseBadRequest(
                    "CommCare HQ no longer supports CSV upload. "
                    "Please convert to Excel 2007 or higher (.xlsx) and try again."
                )
            except UnicodeDecodeError:
                return HttpResponseBadRequest("Unrecognized format")

        try:
            self.user_specs = self.workbook.get_worksheet(title='users')
        except KeyError:
            try:
                self.user_specs = self.workbook.get_worksheet()
            except IndexError:
                return HttpResponseBadRequest("Workbook has no worksheets")

        try:
            self.group_specs = self.workbook.get_worksheet(title='groups')
        except KeyError:
            self.group_specs = []

        try:
            check_headers(self.user_specs)
        except Exception, e:
            return HttpResponseBadRequest(e)
Example #16
0
    def uploaded_file(self):
        try:
            bulk_file = self.request.FILES['bulk_upload_file']
            if bulk_file.size > self.MAX_SIZE:
                raise BulkUploadCasesException(
                    _(u"File size too large. "
                      "Please upload file less than"
                      " {size} Megabytes").format(size=self.MAX_SIZE /
                                                  self.ONE_MB))

        except KeyError:
            raise BulkUploadCasesException(_("No files uploaded"))
        try:
            return WorkbookJSONReader(bulk_file)
        except InvalidFileException:
            try:
                csv.DictReader(
                    io.StringIO(bulk_file.read().decode('utf-8'),
                                newline=None))
                raise BulkUploadCasesException(
                    _("CommCare HQ does not support that file type."
                      "Please convert to Excel 2007 or higher (.xlsx) "
                      "and try again."))
            except UnicodeDecodeError:
                raise BulkUploadCasesException(_("Unrecognized format"))
        except JSONReaderError as e:
            raise BulkUploadCasesException(
                _('Your upload was unsuccessful. %s') % e.message)
Example #17
0
    def __init__(self, task, file_ref_id):
        self.task = task
        self.progress = 0

        if self.task:
            DownloadBase.set_progress(self.task, 0, 100)

        download_ref = DownloadBase.get(file_ref_id)
        self.workbook = WorkbookJSONReader(download_ref.get_filename())
Example #18
0
class FixtureWorkbook(object):
    """
    Helper class for working with the fixture workbook
    """
    def __init__(self, file_or_filename):
        try:
            self.workbook = WorkbookJSONReader(file_or_filename)
        except AttributeError:
            raise FixtureUploadError(
                _("Error processing your Excel (.xlsx) file"))
        except Exception:
            raise FixtureUploadError(
                _("Invalid file-format. Please upload a valid xlsx file."))

    def get_types_sheet(self):
        try:
            return self.workbook.get_worksheet(title='types')
        except WorksheetNotFound as e:
            raise FixtureUploadError(
                _("Workbook does not contain a sheet called '%(title)s'") %
                {'title': e.title})

    def get_data_sheet(self, data_type):
        return self.workbook.get_worksheet(data_type.tag)

    def get_all_type_sheets(self):
        type_sheets = []
        seen_tags = set()
        for number_of_fixtures, dt in enumerate(self.get_types_sheet()):
            table_definition = FixtureTableDefinition.from_row(dt)
            if table_definition.table_id in seen_tags:
                raise DuplicateFixtureTagException(
                    _(FAILURE_MESSAGES['duplicate_tag']).format(
                        tag=table_definition.table_id))

            seen_tags.add(table_definition.table_id)
            type_sheets.append(table_definition)
        return type_sheets

    def validate(self):
        self.get_types_sheet()
        self.get_all_type_sheets()
Example #19
0
    def post(self, request, *args, **kwargs):
        upload = request.FILES.get("bulk_upload_file")
        """View's dispatch method automatically calls this"""
        try:
            self.workbook = WorkbookJSONReader(upload)
        except InvalidFileException:
            try:
                csv.DictReader(io.StringIO(upload.read().decode("ascii"), newline=None))
                return HttpResponseBadRequest(
                    "CommCare HQ no longer supports CSV upload. "
                    "Please convert to Excel 2007 or higher (.xlsx) "
                    "and try again."
                )
            except UnicodeDecodeError:
                return HttpResponseBadRequest("Unrecognized format")
        except JSONReaderError as e:
            messages.error(request, "Your upload was unsuccessful. %s" % e.message)
            return self.get(request, *args, **kwargs)
        except HeaderValueError as e:
            return HttpResponseBadRequest("Upload encountered a data type error: %s" % e.message)

        try:
            self.user_specs = self.workbook.get_worksheet(title="users")
        except WorksheetNotFound:
            try:
                self.user_specs = self.workbook.get_worksheet()
            except WorksheetNotFound:
                return HttpResponseBadRequest("Workbook has no worksheets")

        try:
            self.group_specs = self.workbook.get_worksheet(title="groups")
        except WorksheetNotFound:
            self.group_specs = []

        self.location_specs = []
        if Domain.get_by_name(self.domain).commtrack_enabled:
            try:
                self.location_specs = self.workbook.get_worksheet(title="locations")
            except WorksheetNotFound:
                # if there is no sheet for locations (since this was added
                # later and is optional) we don't error
                pass

        try:
            check_headers(self.user_specs)
        except UserUploadError as e:
            return HttpResponseBadRequest(e)

        task_ref = expose_cached_download(None, expiry=1 * 60 * 60)
        task = bulk_upload_async.delay(
            self.domain, list(self.user_specs), list(self.group_specs), list(self.location_specs)
        )
        task_ref.set_task(task)
        return HttpResponseRedirect(reverse(UserUploadStatusView.urlname, args=[self.domain, task_ref.download_id]))
    def setUpClass(cls):
        cls.app = Application.wrap(cls.get_json("app"))
        # Todo, refactor this into BulkAppTranslationTestBase.upload_raw_excel_translations
        file = StringIO()
        export_raw(cls.excel_headers, cls.excel_data, file, format=Format.XLS_2007)

        with tempfile.TemporaryFile(suffix='.xlsx') as f:
            f.write(file.getvalue())
            wb_reader = WorkbookJSONReader(f)
            cls.expected_workbook = [{'name': ws.title, 'rows': list(ws)}
                                     for ws in wb_reader.worksheets]
Example #21
0
def import_locations_async(domain, file_ref_id):
    task = import_locations_async

    DownloadBase.set_progress(task, 0, 100)
    download_ref = DownloadBase.get(file_ref_id)
    workbook = WorkbookJSONReader(download_ref.get_filename())
    worksheets = workbook.worksheets

    results = list(import_locations(domain, worksheets, task))

    DownloadBase.set_progress(task, 100, 100)

    return {'messages': results}
Example #22
0
    def post(self, request):
        """View's dispatch method automatically calls this"""
        redirect = request.POST.get('redirect')

        try:
            self.workbook = WorkbookJSONReader(request.file)
        except InvalidFileException:
            try:
                csv.DictReader(
                    io.StringIO(request.file.read().decode('ascii'),
                                newline=None))
                return HttpResponseBadRequest(
                    "CommCare HQ no longer supports CSV upload. "
                    "Please convert to Excel 2007 or higher (.xlsx) "
                    "and try again.")
            except UnicodeDecodeError:
                return HttpResponseBadRequest("Unrecognized format")
        except JSONReaderError as e:
            messages.error(request,
                           'Your upload was unsuccessful. %s' % e.message)
            return HttpResponseRedirect(redirect)

        try:
            self.user_specs = self.workbook.get_worksheet(title='users')
        except WorksheetNotFound:
            try:
                self.user_specs = self.workbook.get_worksheet()
            except WorksheetNotFound:
                return HttpResponseBadRequest("Workbook has no worksheets")

        try:
            self.group_specs = self.workbook.get_worksheet(title='groups')
        except WorksheetNotFound:
            self.group_specs = []

        try:
            check_headers(self.user_specs)
        except Exception, e:
            return HttpResponseBadRequest(e)
Example #23
0
class FixtureWorkbook(object):
    """
    Helper class for working with the fixture workbook
    """

    def __init__(self, file_or_filename):
        try:
            self.workbook = WorkbookJSONReader(file_or_filename)
        except AttributeError:
            raise FixtureUploadError(_("Error processing your Excel (.xlsx) file"))
        except Exception:
            raise FixtureUploadError(_("Invalid file-format. Please upload a valid xlsx file."))


    def get_types_sheet(self):
        try:
            return self.workbook.get_worksheet(title='types')
        except WorksheetNotFound as e:
            raise FixtureUploadError(_("Workbook does not contain a sheet called '%(title)s'") % {'title': e.title})

    def get_data_sheet(self, data_type):
        return self.workbook.get_worksheet(data_type.tag)

    def get_all_type_sheets(self):
        type_sheets = []
        seen_tags = set()
        for number_of_fixtures, dt in enumerate(self.get_types_sheet()):
            table_definition = FixtureTableDefinition.from_row(dt)
            if table_definition.table_id in seen_tags:
                raise DuplicateFixtureTagException(
                    _(FAILURE_MESSAGES['duplicate_tag']).format(tag=table_definition.table_id))

            seen_tags.add(table_definition.table_id)
            type_sheets.append(table_definition)
        return type_sheets

    def validate(self):
        self.get_types_sheet()
        self.get_all_type_sheets()
Example #24
0
 def uploaded_file(self):
     try:
         bulk_file = self.request.FILES['bulk_upload_file']
     except KeyError:
         raise BulkUploadCasesException(_("No files uploaded"))
     try:
         return WorkbookJSONReader(bulk_file)
     except InvalidFileException:
         try:
             csv.DictReader(io.StringIO(bulk_file.read().decode('ascii'),
                                        newline=None))
             raise BulkUploadCasesException(_("CommCare HQ no longer supports CSV upload. "
                                              "Please convert to Excel 2007 or higher (.xlsx) "
                                              "and try again."))
         except UnicodeDecodeError:
             raise BulkUploadCasesException(_("Unrecognized format"))
     except JSONReaderError as e:
         raise BulkUploadCasesException(_('Your upload was unsuccessful. %s') % e.message)
Example #25
0
    def post(self, request):
        """View's dispatch method automatically calls this"""
        redirect = request.POST.get('redirect')

        try:
            self.workbook = WorkbookJSONReader(request.file)
        except InvalidFileException:
            try:
                csv.DictReader(io.StringIO(request.file.read().decode('ascii'),
                                           newline=None))
                return HttpResponseBadRequest(
                    "CommCare HQ no longer supports CSV upload. "
                    "Please convert to Excel 2007 or higher (.xlsx) "
                    "and try again."
                )
            except UnicodeDecodeError:
                return HttpResponseBadRequest("Unrecognized format")
        except JSONReaderError as e:
            messages.error(request,
                           'Your upload was unsuccessful. %s' % e.message)
            return HttpResponseRedirect(redirect)

        try:
            self.user_specs = self.workbook.get_worksheet(title='users')
        except WorksheetNotFound:
            try:
                self.user_specs = self.workbook.get_worksheet()
            except WorksheetNotFound:
                return HttpResponseBadRequest("Workbook has no worksheets")

        try:
            self.group_specs = self.workbook.get_worksheet(title='groups')
        except WorksheetNotFound:
            self.group_specs = []

        try:
            check_headers(self.user_specs)
        except Exception, e:
            return HttpResponseBadRequest(e)
Example #26
0
class UploadCommCareUsers(BaseManageCommCareUserView):
    template_name = 'users/upload_commcare_users.html'
    urlname = 'upload_commcare_users'
    page_title = ugettext_noop("Bulk Upload Mobile Workers")

    @method_decorator(requires_privilege_with_fallback(privileges.BULK_USER_MANAGEMENT))
    def dispatch(self, request, *args, **kwargs):
        return super(UploadCommCareUsers, self).dispatch(request, *args, **kwargs)

    @property
    def page_context(self):
        context = {
            'bulk_upload': {
                "help_site": {
                    "address": BULK_MOBILE_HELP_SITE,
                    "name": _("CommCare Help Site"),
                },
                "download_url": reverse(
                    "download_commcare_users", args=(self.domain,)),
                "adjective": _("mobile worker"),
                "plural_noun": _("mobile workers"),
            },
            'show_secret_settings': self.request.REQUEST.get("secret", False),
        }
        context.update({
            'bulk_upload_form': get_bulk_upload_form(context),
        })
        return context

    def post(self, request, *args, **kwargs):
        """View's dispatch method automatically calls this"""
        upload = request.FILES.get('bulk_upload_file')
        try:
            self.workbook = WorkbookJSONReader(upload)
        except InvalidFileException:
            try:
                csv.DictReader(io.StringIO(upload.read().decode('ascii'),
                                           newline=None))
                return HttpResponseBadRequest(
                    "CommCare HQ no longer supports CSV upload. "
                    "Please convert to Excel 2007 or higher (.xlsx) "
                    "and try again."
                )
            except UnicodeDecodeError:
                return HttpResponseBadRequest("Unrecognized format")
        except JSONReaderError as e:
            messages.error(request,
                           'Your upload was unsuccessful. %s' % e.message)
            return self.get(request, *args, **kwargs)
        except HeaderValueError as e:
            return HttpResponseBadRequest("Upload encountered a data type error: %s"
                                          % e.message)

        try:
            self.user_specs = self.workbook.get_worksheet(title='users')
        except WorksheetNotFound:
            try:
                self.user_specs = self.workbook.get_worksheet()
            except WorksheetNotFound:
                return HttpResponseBadRequest("Workbook has no worksheets")

        try:
            self.group_specs = self.workbook.get_worksheet(title='groups')
        except WorksheetNotFound:
            self.group_specs = []

        self.location_specs = []
        if Domain.get_by_name(self.domain).commtrack_enabled:
            try:
                self.location_specs = self.workbook.get_worksheet(title='locations')
            except WorksheetNotFound:
                # if there is no sheet for locations (since this was added
                # later and is optional) we don't error
                pass

        try:
            check_headers(self.user_specs)
        except UserUploadError as e:
            return HttpResponseBadRequest(e)

        task_ref = expose_cached_download(None, expiry=1*60*60)
        task = bulk_upload_async.delay(
            self.domain,
            list(self.user_specs),
            list(self.group_specs),
            list(self.location_specs)
        )
        task_ref.set_task(task)
        return HttpResponseRedirect(
            reverse(
                UserUploadStatusView.urlname,
                args=[self.domain, task_ref.download_id]
            )
        )
Example #27
0
    def clean_message_bank_file(self):
        value = self.cleaned_data.get("message_bank_file")

        if not value:
            raise ValidationError(_("Please choose a file."))

        try:
            workbook = WorkbookJSONReader(value)
        except InvalidFileException:
            raise ValidationError(
                _("Invalid format. Please convert to Excel 2007 or higher (.xlsx) and try again."
                  ))

        try:
            worksheet = workbook.get_worksheet()
        except WorksheetNotFound:
            raise ValidationError(_("Workbook has no worksheets."))

        message_ids = {}
        messages = []
        row_num = 2
        for row in worksheet:
            if "ID" not in row:
                raise ValidationError(_("Column 'ID' not found."))
            if "Message" not in row:
                raise ValidationError(_("Column 'Message' not found."))

            msg_id = row.get("ID")
            text = row.get("Message")

            try:
                assert isinstance(msg_id, basestring)
                msg_id = msg_id.strip()
                assert len(msg_id) > 1
                assert msg_id[0].upper() in "ABCDEFGH"
            except Exception:
                raise ValidationError(
                    _("Invalid ID at row %(row_num)s") % {"row_num": row_num})

            if msg_id in message_ids:
                raise ValidationError(
                    _("Duplicate ID at row %(row_num)s") %
                    {"row_num": row_num})

            try:
                assert isinstance(text, basestring)
                text = text.strip()
                assert len(text) > 0
            except Exception:
                raise ValidationError(
                    _("Invalid Message at row %(row_num)s") %
                    {"row_num": row_num})

            try:
                msg_id.encode("ascii")
            except Exception:
                raise ValidationError(
                    _("ID at row %(row_num)s contains invalid character(s)") %
                    {"row_num": row_num})

            try:
                text.encode("ascii")
            except Exception:
                raise ValidationError(
                    _("Message at row %(row_num)s contains invalid character(s)"
                      ) % {"row_num": row_num})

            if len(text) > 160:
                raise ValidationError(
                    _("Message at row %(row_num)s is longer than 160 characters."
                      ) % {"row_num": row_num})

            messages.append({
                "msg_id": msg_id,
                "text": text,
            })
            message_ids[msg_id] = True
            row_num += 1

        return messages
Example #28
0
    def clean_message_bank_file(self):
        value = self.cleaned_data.get("message_bank_file")

        if not value:
            raise ValidationError(_("Please choose a file."))

        try:
            workbook = WorkbookJSONReader(value)
        except InvalidFileException:
            raise ValidationError(_("Invalid format. Please convert to Excel 2007 or higher (.xlsx) and try again."))

        try:
            worksheet = workbook.get_worksheet()
        except WorksheetNotFound:
            raise ValidationError(_("Workbook has no worksheets."))

        message_ids = {}
        messages = []
        row_num = 2
        for row in worksheet:
            if "ID" not in row:
                raise ValidationError(_("Column 'ID' not found."))
            if "Message" not in row:
                raise ValidationError(_("Column 'Message' not found."))

            msg_id = row.get("ID")
            text = row.get("Message")

            try:
                assert isinstance(msg_id, basestring)
                msg_id = msg_id.strip()
                assert len(msg_id) > 1
                assert msg_id[0].upper() in "ABCDEFGH"
            except Exception:
                raise ValidationError(_("Invalid ID at row %(row_num)s") % {"row_num" : row_num})

            if msg_id in message_ids:
                raise ValidationError(_("Duplicate ID at row %(row_num)s") % {"row_num" : row_num})

            try:
                assert isinstance(text, basestring)
                text = text.strip()
                assert len(text) > 0
            except Exception:
                raise ValidationError(_("Invalid Message at row %(row_num)s") % {"row_num" : row_num})

            try:
                msg_id.encode("ascii")
            except Exception:
                raise ValidationError(_("ID at row %(row_num)s contains invalid character(s)") % {"row_num" : row_num})

            try:
                text.encode("ascii")
            except Exception:
                raise ValidationError(_("Message at row %(row_num)s contains invalid character(s)") % {"row_num" : row_num})

            if len(text) > 160:
                raise ValidationError(_("Message at row %(row_num)s is longer than 160 characters.") % {"row_num" : row_num})

            messages.append({
                "msg_id" : msg_id,
                "text" : text,
            })
            message_ids[msg_id] = True
            row_num += 1

        return messages
Example #29
0
def upload_fixture_api(request, domain, **kwargs):
    """
        Use following curl-command to test.
        > curl -v --digest http://127.0.0.1:8000/a/gsid/fixtures/fixapi/ -u [email protected]:password
                -F "file-to-upload=@hqtest_fixtures.xlsx"
                -F "replace=true"
    """
    response_codes = {"fail": 405, "warning": 402, "success": 200}
    error_messages = {
        "invalid_post_req":
        "Invalid post request. Submit the form with field 'file-to-upload' and POST parameter 'replace'",
        "has_no_permission":
        "User {attr} doesn't have permission to upload fixtures",
        "invalid_file":
        "Error processing your file. Submit a valid (.xlsx) file",
        "has_no_sheet":
        "Workbook does not have a sheet called {attr}",
        "unknown_fail":
        "Fixture upload couldn't succeed due to the following error: {attr}",
    }

    def _return_response(code, message):
        resp_json = {}
        resp_json["code"] = code
        resp_json["message"] = message
        return HttpResponse(json.dumps(resp_json), mimetype="application/json")

    try:
        upload_file = request.FILES["file-to-upload"]
        replace = request.POST["replace"]
        if replace.lower() == "true":
            replace = True
        elif replace.lower() == "false":
            replace = False
    except Exception:
        return _return_response(response_codes["fail"],
                                error_messages["invalid_post_req"])

    if not request.couch_user.has_permission(domain,
                                             Permissions.edit_data.name):
        error_message = error_messages["has_no_permission"].format(
            attr=request.couch_user.username)
        return _return_response(response_codes["fail"], error_message)

    try:
        workbook = WorkbookJSONReader(upload_file)
    except Exception:
        return _return_response(response_codes["fail"],
                                error_messages["invalid_file"])

    try:
        upload_resp = run_upload(
            request, domain, workbook,
            replace=replace)  # error handle for other files
    except WorksheetNotFound as e:
        error_message = error_messages["has_no_sheet"].format(attr=e.title)
        return _return_response(response_codes["fail"], error_message)
    except ExcelMalformatException as e:
        notify_exception(request)
        return _return_response(response_codes["fail"], str(e))
    except DuplicateFixtureTagException as e:
        return _return_response(response_codes["fail"], str(e))
    except FixtureAPIException as e:
        return _return_response(response_codes["fail"], str(e))
    except Exception as e:
        notify_exception(request, message=e)
        error_message = error_messages["unknown_fail"].format(attr=e)
        return _return_response(response_codes["fail"], error_message)

    num_unknown_groups = len(upload_resp["unknown_groups"])
    num_unknown_users = len(upload_resp["unknown_users"])
    resp_json = {}

    if not num_unknown_users and not num_unknown_groups:
        num_uploads = upload_resp["number_of_fixtures"]
        success_message = "Successfully uploaded %d fixture%s." % (
            num_uploads, 's' if num_uploads > 1 else '')
        return _return_response(response_codes["success"], success_message)
    else:
        resp_json["code"] = response_codes["warning"]

    warn_groups = "%d group%s unknown" % (num_unknown_groups, 's are'
                                          if num_unknown_groups > 1 else ' is')
    warn_users = "%d user%s unknown" % (num_unknown_users, 's are'
                                        if num_unknown_users > 1 else ' is')
    resp_json["message"] = "Fixtures have been uploaded. But following "
    if num_unknown_groups:
        resp_json["message"] += "%s %s" % (warn_groups,
                                           upload_resp["unknown_groups"])
    if num_unknown_users:
        resp_json["message"] += "%s%s%s" % (
            ("and following " if num_unknown_groups else ""), warn_users,
            upload_resp["unknown_users"])

    return HttpResponse(json.dumps(resp_json), mimetype="application/json")
Example #30
0
    def post(self, request):
        """View's dispatch method automatically calls this"""
        def error_redirect():
            return HttpResponseRedirect(
                reverse("fixture_interface_dispatcher",
                        args=[],
                        kwargs={
                            'domain': self.domain,
                            'report_slug': 'edit_lookup_tables'
                        }))

        try:
            replace = request.POST["replace"]
            replace = True
        except KeyError:
            replace = False

        try:
            workbook = WorkbookJSONReader(request.file)
        except AttributeError:
            messages.error(request,
                           _("Error processing your Excel (.xlsx) file"))
            return error_redirect()
        except Exception as e:
            messages.error(
                request,
                _("Invalid file-format. Please upload a valid xlsx file."))
            return error_redirect()

        try:
            upload_result = run_upload(request,
                                       self.domain,
                                       workbook,
                                       replace=replace)
            if upload_result["unknown_groups"]:
                for group_name in upload_result["unknown_groups"]:
                    messages.error(
                        request,
                        _("Unknown group: '%(name)s'") % {'name': group_name})
            if upload_result["unknown_users"]:
                for user_name in upload_result["unknown_users"]:
                    messages.error(
                        request,
                        _("Unknown user: '******'") % {'name': user_name})
        except WorksheetNotFound as e:
            messages.error(
                request,
                _("Workbook does not contain a sheet called '%(title)s'") %
                {'title': e.title})
            return error_redirect()
        except ExcelMalformatException as e:
            messages.error(
                request,
                _("Uploaded excel file has following formatting-problems: '%(e)s'"
                  ) % {'e': e})
            return error_redirect()
        except DuplicateFixtureTagException as e:
            messages.error(request, e)
        except FixtureAPIException as e:
            messages.error(request, _(str(e)))
            return error_redirect()
        except Exception as e:
            notify_exception(request, message=str(e))
            messages.error(request, str(e))
            messages.error(
                request,
                _("Fixture upload failed for some reason and we have noted this failure. "
                  "Please make sure the excel file is correctly formatted and try again."
                  ))
            return error_redirect()

        return HttpResponseRedirect(
            reverse("fixture_interface_dispatcher",
                    args=[],
                    kwargs={
                        'domain': self.domain,
                        'report_slug': 'edit_lookup_tables'
                    }))
Example #31
0
    def test_archive_forms_wrong_domain(self):
        uploaded_file = WorkbookJSONReader(join(BASE_PATH, BASIC_XLSX))

        response = archive_forms('wrong_domain', self.user, list(uploaded_file.get_worksheet()))

        self.assertEqual(len(response['errors']), len(self.xforms), "Error when wrong domain")
Example #32
0
def process_bulk_app_translation_upload(app, f):
    """
    Process the bulk upload file for the given app.
    We return these message tuples instead of calling them now to allow this
    function to be used independently of request objects.

    :param app:
    :param f:
    :return: Returns a list of message tuples. The first item in each tuple is
    a function like django.contrib.messages.error, and the second is a string.
    """
    def none_or_unicode(val):
        return unicode(val) if val is not None else val

    msgs = []

    headers = expected_bulk_app_sheet_headers(app)
    expected_sheets = {h[0]: h[1] for h in headers}
    processed_sheets = set()

    try:
        workbook = WorkbookJSONReader(f)
    except (HeaderValueError, InvalidFileException) as e:
        msgs.append((
            messages.error,
            _("App Translation Failed! "
              "Please make sure you are using a valid Excel 2007 or later (.xlsx) file. "
              "Error details: {}.").format(e)))
        return msgs

    for sheet in workbook.worksheets:
        # sheet.__iter__ can only be called once, so cache the result
        rows = [row for row in sheet]
        # Convert every key and value to a string
        for i in xrange(len(rows)):
            rows[i] = {
                unicode(k): none_or_unicode(v)
                for k, v in rows[i].iteritems()
            }

        # CHECK FOR REPEAT SHEET
        if sheet.worksheet.title in processed_sheets:
            msgs.append(
                (messages.error, 'Sheet "%s" was repeated. Only the first ' +
                 'occurrence has been processed' % sheet.worksheet.title))
            continue

        # CHECK FOR BAD SHEET NAME
        expected_columns = expected_sheets.get(sheet.worksheet.title, None)
        if expected_columns is None:
            msgs.append((messages.error,
                         'Skipping sheet "%s", did not recognize title' %
                         sheet.worksheet.title))
            continue

        # CHECK FOR MISSING KEY COLUMN
        if sheet.worksheet.title == "Modules and Forms":
            # Several columns on this sheet could be used to uniquely identify
            # rows. Using sheet_name for now, but unique_id could also be used.
            if expected_columns[1] not in sheet.headers:
                msgs.append(
                    (messages.error,
                     'Skipping sheet "%s", could not find "%s" column' %
                     (sheet.worksheet.title, expected_columns[1])))
                continue
        elif expected_columns[0] == "case_property":
            # It's a module sheet
            if (expected_columns[0] not in sheet.headers
                    or expected_columns[1] not in sheet.headers):
                msgs.append(
                    (messages.error,
                     'Skipping sheet "%s", could not find case_property'
                     ' or list_or_detail column.' % sheet.worksheet.title))
                continue
        else:
            # It's a form sheet
            if expected_columns[0] not in sheet.headers:
                msgs.append(
                    (messages.error,
                     'Skipping sheet "%s", could not find label column' %
                     sheet.worksheet.title))
                continue

        processed_sheets.add(sheet.worksheet.title)

        # CHECK FOR MISSING COLUMNS
        missing_cols = set(expected_columns) - set(sheet.headers)
        if len(missing_cols) > 0:
            msgs.append((messages.warning,
                         'Sheet "%s" has less columns than expected. '
                         'Sheet will be processed but the following'
                         ' translations will be unchanged: %s' %
                         (sheet.worksheet.title, " ,".join(missing_cols))))

        # CHECK FOR EXTRA COLUMNS
        extra_cols = set(sheet.headers) - set(expected_columns)
        if len(extra_cols) > 0:
            msgs.append((
                messages.warning, 'Sheet "%s" has unrecognized columns. '
                'Sheet will be processed but ignoring the following columns: %s'
                % (sheet.worksheet.title, " ,".join(extra_cols))))

        # NOTE: At the moment there is no missing row detection.
        # This could be added if we want though
        #      (it is not that bad if a user leaves out a row)

        if sheet.worksheet.title == "Modules_and_forms":
            # It's the first sheet
            ms = process_modules_and_forms_sheet(rows, app)
            msgs.extend(ms)
        elif sheet.headers[0] == "case_property":
            # It's a module sheet
            ms = update_case_list_translations(sheet, rows, app)
            msgs.extend(ms)
        else:
            # It's a form sheet
            ms = update_form_translations(sheet, rows, missing_cols, app)
            msgs.extend(ms)

    msgs.append((messages.success, _("App Translations Updated!")))
    return msgs
Example #33
0
class UploadCommCareUsers(TemplateView):

    template_name = 'users/upload_commcare_users.html'

    def get_context_data(self, **kwargs):
        """TemplateView automatically calls this to render the view (on a get)"""
        context = _users_context(self.request, self.domain)
        context["show_secret_settings"] = self.request.REQUEST.get("secret", False)
        return context

    @method_decorator(get_file)
    def post(self, request):
        """View's dispatch method automatically calls this"""

        try:
            self.workbook = WorkbookJSONReader(request.file)
        except InvalidFileException:
            try:
                csv.DictReader(io.StringIO(request.file.read().decode('ascii'), newline=None))
                return HttpResponseBadRequest(
                    "CommCare HQ no longer supports CSV upload. "
                    "Please convert to Excel 2007 or higher (.xlsx) and try again."
                )
            except UnicodeDecodeError:
                return HttpResponseBadRequest("Unrecognized format")

        try:
            self.user_specs = self.workbook.get_worksheet(title='users')
        except KeyError:
            try:
                self.user_specs = self.workbook.get_worksheet()
            except IndexError:
                return HttpResponseBadRequest("Workbook has no worksheets")

        try:
            self.group_specs = self.workbook.get_worksheet(title='groups')
        except KeyError:
            self.group_specs = []

        try:
            check_headers(self.user_specs)
        except Exception, e:
            return HttpResponseBadRequest(e)

        response = HttpResponse()
        response_writer = csv.DictWriter(response, ['username', 'flag', 'row'])
        response_rows = []
        async = request.REQUEST.get("async", False)
        if async:
            download_id = uuid.uuid4().hex
            bulk_upload_async.delay(download_id, self.domain,
                                    list(self.user_specs),
                                    list(self.group_specs))
            messages.success(request, 
                'Your upload is in progress. You can check the progress at "%s%s".' %  \
                (get_url_base(), reverse('retrieve_download', kwargs={'download_id': download_id})),
                extra_tags="html")
        else:
            ret = create_or_update_users_and_groups(self.domain, self.user_specs, self.group_specs)
            for error in ret["errors"]:
                messages.error(request, error)
            
            for row in ret["rows"]:
                response_writer.writerow(row)
                response_rows.append(row)

        redirect = request.POST.get('redirect')
        if redirect:
            if not async:
                messages.success(request, 'Your bulk user upload is complete!')
            problem_rows = []
            for row in response_rows:
                if row['flag'] not in ('updated', 'created'):
                    problem_rows.append(row)
            if problem_rows:
                messages.error(request, 'However, we ran into problems with the following users:')
                for row in problem_rows:
                    if row['flag'] == 'missing-data':
                        messages.error(request, 'A row with no username was skipped')
                    else:
                        messages.error(request, '{username}: {flag}'.format(**row))
            return HttpResponseRedirect(redirect)
        else:
            return response
Example #34
0
class UploadCommCareUsers(BaseManageCommCareUserView):
    template_name = 'users/upload_commcare_users.html'
    urlname = 'upload_commcare_users'
    page_title = ugettext_noop("Bulk Upload Mobile Workers")

    @property
    def page_context(self):
        return {
            'show_secret_settings': self.request.REQUEST.get("secret", False),
        }

    @method_decorator(get_file)
    def post(self, request, *args, **kwargs):
        """View's dispatch method automatically calls this"""
        redirect = request.POST.get('redirect')

        try:
            self.workbook = WorkbookJSONReader(request.file)
        except InvalidFileException:
            try:
                csv.DictReader(io.StringIO(request.file.read().decode('ascii'),
                                           newline=None))
                return HttpResponseBadRequest(
                    "CommCare HQ no longer supports CSV upload. "
                    "Please convert to Excel 2007 or higher (.xlsx) "
                    "and try again."
                )
            except UnicodeDecodeError:
                return HttpResponseBadRequest("Unrecognized format")
        except JSONReaderError as e:
            messages.error(request,
                           'Your upload was unsuccessful. %s' % e.message)
            return HttpResponseRedirect(redirect)
        except HeaderValueError as e:
            return HttpResponseBadRequest("Upload encountered a data type error: %s"
                                          % e.message)

        try:
            self.user_specs = self.workbook.get_worksheet(title='users')
        except WorksheetNotFound:
            try:
                self.user_specs = self.workbook.get_worksheet()
            except WorksheetNotFound:
                return HttpResponseBadRequest("Workbook has no worksheets")

        try:
            self.group_specs = self.workbook.get_worksheet(title='groups')
        except WorksheetNotFound:
            self.group_specs = []

        self.location_specs = []
        if Domain.get_by_name(self.domain).commtrack_enabled:
            try:
                self.location_specs = self.workbook.get_worksheet(title='locations')
            except WorksheetNotFound:
                # if there is no sheet for locations (since this was added
                # later and is optional) we don't error
                pass

        try:
            check_headers(self.user_specs)
        except UserUploadError as e:
            return HttpResponseBadRequest(e)

        response = HttpResponse()
        response_rows = []
        async = request.REQUEST.get("async", False)
        if async:
            download_id = uuid.uuid4().hex
            bulk_upload_async.delay(
                download_id,
                self.domain,
                list(self.user_specs),
                list(self.group_specs),
                list(self.location_specs)
            )
            messages.success(request,
                'Your upload is in progress. You can check the progress <a href="%s">here</a>.' %\
                reverse('hq_soil_download', kwargs={'domain': self.domain, 'download_id': download_id}),
                extra_tags="html")
        else:
            ret = create_or_update_users_and_groups(
                self.domain,
                self.user_specs,
                self.group_specs,
                self.location_specs
            )
            for error in ret["errors"]:
                messages.error(request, error)

            for row in ret["rows"]:
                response_rows.append(row)

        if redirect:
            if not async:
                messages.success(request,
                                 _('Your bulk user upload is complete!'))
            problem_rows = []
            for row in response_rows:
                if row['flag'] not in ('updated', 'created'):
                    problem_rows.append(row)
            if problem_rows:
                messages.error(
                    request,
                    _('However, we ran into problems with the following users:')
                )
                for row in problem_rows:
                    if row['flag'] == 'missing-data':
                        messages.error(request,
                                       _('A row with no username was skipped'))
                    else:
                        messages.error(request,
                                       '{username}: {flag}'.format(**row))
            return HttpResponseRedirect(redirect)
        else:
            return response
Example #35
0
class UploadCommCareUsers(BaseManageCommCareUserView):
    template_name = 'users/upload_commcare_users.html'
    urlname = 'upload_commcare_users'
    page_title = ugettext_noop("Bulk Upload Mobile Workers")

    @method_decorator(requires_privilege_with_fallback(privileges.BULK_USER_MANAGEMENT))
    def dispatch(self, request, *args, **kwargs):
        return super(UploadCommCareUsers, self).dispatch(request, *args, **kwargs)

    @property
    def page_context(self):
        context = {
            'bulk_upload': {
                "help_site": {
                    "address": BULK_MOBILE_HELP_SITE,
                    "name": _("CommCare Help Site"),
                },
                "download_url": reverse(
                    "download_commcare_users", args=(self.domain,)),
                "adjective": _("mobile worker"),
                "plural_noun": _("mobile workers"),
            },
            'show_secret_settings': self.request.REQUEST.get("secret", False),
        }
        context.update({
            'bulk_upload_form': get_bulk_upload_form(context),
        })
        return context

    def post(self, request, *args, **kwargs):
        upload = request.FILES.get('bulk_upload_file')
        """View's dispatch method automatically calls this"""
        try:
            self.workbook = WorkbookJSONReader(upload)
        except InvalidFileException:
            try:
                csv.DictReader(io.StringIO(upload.read().decode('ascii'),
                                           newline=None))
                return HttpResponseBadRequest(
                    "CommCare HQ no longer supports CSV upload. "
                    "Please convert to Excel 2007 or higher (.xlsx) "
                    "and try again."
                )
            except UnicodeDecodeError:
                return HttpResponseBadRequest("Unrecognized format")
        except JSONReaderError as e:
            messages.error(request,
                           'Your upload was unsuccessful. %s' % e.message)
            return self.get(request, *args, **kwargs)
        except HeaderValueError as e:
            return HttpResponseBadRequest("Upload encountered a data type error: %s"
                                          % e.message)

        try:
            self.user_specs = self.workbook.get_worksheet(title='users')
        except WorksheetNotFound:
            try:
                self.user_specs = self.workbook.get_worksheet()
            except WorksheetNotFound:
                return HttpResponseBadRequest("Workbook has no worksheets")

        try:
            self.group_specs = self.workbook.get_worksheet(title='groups')
        except WorksheetNotFound:
            self.group_specs = []

        self.location_specs = []
        if Domain.get_by_name(self.domain).commtrack_enabled:
            try:
                self.location_specs = self.workbook.get_worksheet(title='locations')
            except WorksheetNotFound:
                # if there is no sheet for locations (since this was added
                # later and is optional) we don't error
                pass

        try:
            check_headers(self.user_specs)
        except UserUploadError as e:
            return HttpResponseBadRequest(e)

        task_ref = expose_download(None, expiry=1*60*60)
        task = bulk_upload_async.delay(
            self.domain,
            list(self.user_specs),
            list(self.group_specs),
            list(self.location_specs)
        )
        task_ref.set_task(task)
        return HttpResponseRedirect(
            reverse(
                UserUploadStatusView.urlname,
                args=[self.domain, task_ref.download_id]
            )
        )
Example #36
0
    def post(self, request):
        """View's dispatch method automatically calls this"""
        redirect = request.POST.get('redirect')

        try:
            self.workbook = WorkbookJSONReader(request.file)
        except InvalidFileException:
            try:
                csv.DictReader(io.StringIO(request.file.read().decode('ascii'),
                                           newline=None))
                return HttpResponseBadRequest(
                    "CommCare HQ no longer supports CSV upload. "
                    "Please convert to Excel 2007 or higher (.xlsx) "
                    "and try again."
                )
            except UnicodeDecodeError:
                return HttpResponseBadRequest("Unrecognized format")
        except JSONReaderError as e:
            messages.error(request,
                           'Your upload was unsuccessful. %s' % e.message)
            return HttpResponseRedirect(redirect)

        try:
            self.user_specs = self.workbook.get_worksheet(title='users')
        except WorksheetNotFound:
            try:
                self.user_specs = self.workbook.get_worksheet()
            except WorksheetNotFound:
                return HttpResponseBadRequest("Workbook has no worksheets")

        try:
            self.group_specs = self.workbook.get_worksheet(title='groups')
        except WorksheetNotFound:
            self.group_specs = []

        try:
            check_headers(self.user_specs)
        except UserUploadError as e:
            return HttpResponseBadRequest(e)

        response = HttpResponse()
        response_rows = []
        async = request.REQUEST.get("async", False)
        if async:
            download_id = uuid.uuid4().hex
            bulk_upload_async.delay(download_id, self.domain,
                list(self.user_specs),
                list(self.group_specs))
            messages.success(request,
                'Your upload is in progress. You can check the progress <a href="%s">here</a>.' %\
                reverse('hq_soil_download', kwargs={'domain': self.domain, 'download_id': download_id}),
                extra_tags="html")
        else:
            ret = create_or_update_users_and_groups(self.domain, self.user_specs, self.group_specs)
            for error in ret["errors"]:
                messages.error(request, error)

            for row in ret["rows"]:
                response_rows.append(row)

        if redirect:
            if not async:
                messages.success(request,
                                 _('Your bulk user upload is complete!'))
            problem_rows = []
            for row in response_rows:
                if row['flag'] not in ('updated', 'created'):
                    problem_rows.append(row)
            if problem_rows:
                messages.error(
                    request,
                    _('However, we ran into problems with the following users:')
                )
                for row in problem_rows:
                    if row['flag'] == 'missing-data':
                        messages.error(request,
                                       _('A row with no username was skipped'))
                    else:
                        messages.error(request,
                                       '{username}: {flag}'.format(**row))
            return HttpResponseRedirect(redirect)
        else:
            return response
Example #37
0
    def post(self, request, *args, **kwargs):
        """View's dispatch method automatically calls this"""
        redirect = request.POST.get("redirect")

        try:
            self.workbook = WorkbookJSONReader(request.file)
        except InvalidFileException:
            try:
                csv.DictReader(io.StringIO(request.file.read().decode("ascii"), newline=None))
                return HttpResponseBadRequest(
                    "CommCare HQ no longer supports CSV upload. "
                    "Please convert to Excel 2007 or higher (.xlsx) "
                    "and try again."
                )
            except UnicodeDecodeError:
                return HttpResponseBadRequest("Unrecognized format")
        except JSONReaderError as e:
            messages.error(request, "Your upload was unsuccessful. %s" % e.message)
            return HttpResponseRedirect(redirect)
        except HeaderValueError as e:
            return HttpResponseBadRequest("Upload encountered a data type error: %s" % e.message)

        try:
            self.user_specs = self.workbook.get_worksheet(title="users")
        except WorksheetNotFound:
            try:
                self.user_specs = self.workbook.get_worksheet()
            except WorksheetNotFound:
                return HttpResponseBadRequest("Workbook has no worksheets")

        try:
            self.group_specs = self.workbook.get_worksheet(title="groups")
        except WorksheetNotFound:
            self.group_specs = []

        self.location_specs = []
        if Domain.get_by_name(self.domain).commtrack_enabled:
            try:
                self.location_specs = self.workbook.get_worksheet(title="locations")
            except WorksheetNotFound:
                # if there is no sheet for locations (since this was added
                # later and is optional) we don't error
                pass

        try:
            check_headers(self.user_specs)
        except UserUploadError as e:
            return HttpResponseBadRequest(e)

        response = HttpResponse()
        response_rows = []
        async = request.REQUEST.get("async", False)
        if async:
            download_id = uuid.uuid4().hex
            bulk_upload_async.delay(
                download_id, self.domain, list(self.user_specs), list(self.group_specs), list(self.location_specs)
            )
            messages.success(
                request,
                'Your upload is in progress. You can check the progress <a href="%s">here</a>.'
                % reverse("hq_soil_download", kwargs={"domain": self.domain, "download_id": download_id}),
                extra_tags="html",
            )
        else:
            ret = create_or_update_users_and_groups(self.domain, self.user_specs, self.group_specs, self.location_specs)
            for error in ret["errors"]:
                messages.error(request, error)

            for row in ret["rows"]:
                response_rows.append(row)

        if redirect:
            if not async:
                messages.success(request, _("Your bulk user upload is complete!"))
            problem_rows = []
            for row in response_rows:
                if row["flag"] not in ("updated", "created"):
                    problem_rows.append(row)
            if problem_rows:
                messages.error(request, _("However, we ran into problems with the following users:"))
                for row in problem_rows:
                    if row["flag"] == "missing-data":
                        messages.error(request, _("A row with no username was skipped"))
                    else:
                        messages.error(request, "{username}: {flag}".format(**row))
            return HttpResponseRedirect(redirect)
        else:
            return response