示例#1
0
    def _consolidate(self):
        """
        Consolidate valid TransitionRow requests into valid Transition objects
        In case of multiple TransitionRow requests, like in case of merge/split,
        combine them into one Transition
        """
        for location_type_code, rows_for_site_code in self.transition_rows.items():
            for site_code, rows in rows_for_site_code.items():
                errors = []
                for row in rows:
                    errors.extend(row.validate())
                if errors:
                    self.errors.extend(errors)
                    continue

                operation = self._valid_unique_operation(site_code, rows)
                if not operation:
                    continue

                transition = self._consolidated_transition(location_type_code, operation, rows)
                if not transition:
                    continue
                if self._is_valid_transition(transition):
                    self.site_codes_to_be_deprecated.update(transition.old_site_codes)
                    self.valid_transitions[location_type_code].append(transition)
                    for old_username, new_username in transition.user_transitions.items():
                        if old_username:
                            self.usernames.add(normalize_username(old_username, self.domain))
                        if new_username:
                            self.usernames.add(normalize_username(new_username, self.domain))

                # keep note of transition details for consolidated validations
                self.transiting_site_codes.update(transition.old_site_codes)
                self.transiting_site_codes.update(transition.new_site_codes)
                self.new_site_codes_for_location_type[location_type_code].update(transition.new_site_codes)
示例#2
0
def update_usercase(domain, old_username, new_username):
    if "@" not in old_username:
        old_username = normalize_username(old_username, domain)
    if "@" not in new_username:
        new_username = normalize_username(new_username, domain)
    old_user = CouchUser.get_by_username(old_username)
    new_user = CouchUser.get_by_username(new_username)
    if old_user and new_user and old_user.is_commcare_user(
    ) and new_user.is_commcare_user():
        old_user_usercase = old_user.get_usercase()
        new_user_usercase = new_user.get_usercase()
        # pick values that are not already present on the new user's usercase, populated already via HQ
        updates = {}
        for key in set(old_user_usercase.case_json.keys()) - set(
                new_user_usercase.case_json.keys()):
            updates[key] = old_user_usercase.case_json[key]
        if updates:
            case_block = CaseBlock(new_user_usercase.case_id,
                                   update=old_user_usercase.case_json,
                                   user_id=SYSTEM_USER_ID).as_text()
            submit_case_blocks([case_block], domain, user_id=SYSTEM_USER_ID)
    else:
        raise InvalidUserTransition(
            "Invalid Transition with old user %s and new user %s" %
            (old_username, new_username))
示例#3
0
 def validate_spec(self, spec):
     username = spec.get('username')
     if username:
         try:
             normalize_username(str(username), self.domain)
         except TypeError:
             pass
         except ValidationError:
             return self.error_message
示例#4
0
def create_or_update_locations(domain, location_specs, log):
    location_cache = LocationCache()
    users = {}
    for row in location_specs:
        username = row.get('username')
        try:
            username = normalize_username(username, domain)
        except ValidationError:
            log['errors'].append(_("Username must be a valid email address: %s") % username)
        else:
            location_code = unicode(row.get('location-sms-code'))
            if username in users:
                user_mapping = users[username]
            else:
                user_mapping = UserLocMapping(username, domain, location_cache)
                users[username] = user_mapping

            if row.get('remove') == 'y':
                user_mapping.to_remove.add(location_code)
            else:
                user_mapping.to_add.add(location_code)

    for username, mapping in users.iteritems():
        try:
            messages = mapping.save()
            log['errors'].extend(messages)
        except UserUploadError as e:
            log['errors'].append(_('Unable to update locations for {user} because {message}'.format(
                user=username, message=e
            )))
示例#5
0
def create_or_update_locations(domain, location_specs, log):
    location_cache = LocationCache()
    users = {}
    for row in location_specs:
        username = row.get('username')
        try:
            username = normalize_username(username, domain)
        except ValidationError:
            log['errors'].append(
                _("Username must be a valid email address: %s") % username)
        else:
            location_code = unicode(row.get('location-sms-code'))
            if username in users:
                user_mapping = users[username]
            else:
                user_mapping = UserLocMapping(username, domain, location_cache)
                users[username] = user_mapping

            if row.get('remove') == 'y':
                user_mapping.to_remove.add(location_code)
            else:
                user_mapping.to_add.add(location_code)

    for username, mapping in users.iteritems():
        try:
            messages = mapping.save()
            log['errors'].extend(messages)
        except UserUploadError as e:
            log['errors'].append(
                _('Unable to update locations for {user} because {message}'.
                  format(user=username, message=e)))
 def make_mobile_worker(cls, username, domain=None):
     domain = domain or cls.domain
     user = CommCareUser(username=normalize_username(username, domain),
                         domain=domain)
     user.domain_membership = DomainMembership(domain=domain)
     UserESFake.save_doc(user._doc)
     return user
示例#7
0
def _process_user_ownership(di, old_data_item, transaction):
    """Removes users from data items and add new ones based on the user column.

    Note the removal does not happen within a context of the "transaction"
    """
    errors = []
    domain = old_data_item.domain

    old_users = old_data_item.users
    for user in old_users:
        old_data_item.remove_user(user)

    for raw_username in di.get('user', []):
        try:
            username = normalize_username(str(raw_username), domain)
        except ValidationError:
            errors.append(
                _("Invalid username: '******'. Row is not added")
                % {'name': raw_username}
            )
            continue
        user = CommCareUser.get_by_username(username)
        if user:
            old_data_item.add_user(user)
        else:
            errors.append(
                _("Unknown user: '******'. But the row is successfully added")
                % {'name': raw_username}
            )

    return errors
示例#8
0
def get_username(xml):
    try:
        username = re.search(
            r'<[Mm]eta>.*<username>(.*)</username>.*</[Mm]eta>', xml).group(1)
        return normalize_username(username)
    except:
        return None
    def update_cases(self, domain, case_type, user_id):
        case_ids = self.find_case_ids_by_type(domain, case_type)
        accessor = CaseAccessors(domain)
        case_blocks = []
        skip_count = 0
        for case in accessor.iter_cases(case_ids):
            username_of_associated_mobile_workers = case.get_case_property(
                'username')
            try:
                normalized_username = normalize_username(
                    username_of_associated_mobile_workers, domain)
            except ValidationError:
                skip_count += 1
                continue
            user_id_of_mobile_worker = username_to_user_id(normalized_username)
            if user_id_of_mobile_worker:
                case_blocks.append(
                    self.case_block(case, user_id_of_mobile_worker))
            else:
                skip_count += 1
        print(
            f"{len(case_blocks)} to update in {domain}, {skip_count} cases have skipped due to unknown username."
        )

        total = 0
        for chunk in chunked(case_blocks, BATCH_SIZE):
            submit_case_blocks(chunk,
                               domain,
                               device_id=DEVICE_ID,
                               user_id=user_id)
            total += len(chunk)
            print("Updated {} cases on domain {}".format(total, domain))
    def test_add_hq_user_id_to_case(self):
        username = normalize_username("mobile_worker", self.domain)
        new_mobile_worker = CommCareUser.create(self.domain, username, "123", None, None)
        user_id = new_mobile_worker.user_id
        new_mobile_worker.save()

        checkin_case_id = uuid.uuid4().hex
        self.submit_case_block(
            True, checkin_case_id, user_id=user_id, case_type='checkin',
            update={"username": new_mobile_worker.raw_username, "hq_user_id": None}
        )
        checkin_case_no_username_id = uuid.uuid4().hex
        self.submit_case_block(
            True, checkin_case_no_username_id, user_id=user_id, case_type='checkin', update={"hq_user_id": None}
        )
        lab_result_case_id = uuid.uuid4().hex
        self.submit_case_block(
            True, lab_result_case_id, user_id=user_id, case_type='lab_result',
            update={"username": new_mobile_worker.raw_username, "hq_user_id": None}
        )
        checkin_case = self.case_accessor.get_case(checkin_case_id)
        self.assertEqual('', checkin_case.get_case_property('hq_user_id'))
        self.assertEqual(checkin_case.username, 'mobile_worker')

        call_command('add_hq_user_id_to_case', self.domain, 'checkin')

        checkin_case = self.case_accessor.get_case(checkin_case_id)
        checkin_case_no_username = self.case_accessor.get_case(checkin_case_no_username_id)
        lab_result_case = self.case_accessor.get_case(lab_result_case_id)
        self.assertEqual(checkin_case.get_case_property('hq_user_id'), user_id)
        self.assertEqual(checkin_case_no_username.hq_user_id, '')
        self.assertEqual(lab_result_case.hq_user_id, '')
示例#11
0
def _process_user_ownership(di, old_data_item, transaction):
    """Removes users from data items and add new ones based on the user column.

    Note the removal does not happen within a context of the "transaction"
    """
    errors = []
    domain = old_data_item.domain

    old_users = old_data_item.users
    for user in old_users:
        old_data_item.remove_user(user)

    for raw_username in di.get('user', []):
        try:
            username = normalize_username(str(raw_username), domain)
        except ValidationError:
            errors.append(
                _("Invalid username: '******'. Row is not added")
                % {'name': raw_username}
            )
            continue
        user = CommCareUser.get_by_username(username)
        if user:
            old_data_item.add_user(user)
        else:
            errors.append(
                _("Unknown user: '******'. But the row is successfully added")
                % {'name': raw_username}
            )

    return errors
示例#12
0
    def setUp(self):
        super().setUp()
        delete_all_users()

        self.domain_obj = create_domain(self.domain)
        enable_usercase(self.domain)

        with trap_extra_setup(ConnectionError):
            self.es = get_es_new()
            initialize_index_and_mapping(self.es, CASE_SEARCH_INDEX_INFO)

        username = normalize_username("mobile_worker_1", self.domain)
        self.mobile_worker = CommCareUser.create(self.domain, username, "123",
                                                 None, None)
        sync_user_cases(self.mobile_worker)

        self.checkin_case = CaseFactory(self.domain).create_case(
            case_type="checkin",
            owner_id=self.mobile_worker.get_id,
            update={"username": self.mobile_worker.raw_username},
        )
        send_to_elasticsearch(
            "case_search",
            transform_case_for_elasticsearch(self.checkin_case.to_json()))
        self.es.indices.refresh(CASE_SEARCH_INDEX_INFO.index)

        self.case_accessor = CaseAccessors(self.domain)
示例#13
0
    def validate_spec(self, spec):
        try:
            username = normalize_username(spec.get('username'), self.domain)
        except ValidationError:
            return

        if username in self.existing_usernames:
            return self.error_message
示例#14
0
 def make_mobile_worker(cls, username, domain=None, metadata=None):
     user = CommCareUser.create(
         domain=domain or cls.domain,
         username=normalize_username(username),
         password="******",
         created_by=None,
         created_via=None,
         metadata=metadata,
     )
     return user
示例#15
0
 def make_mobile_worker(cls, username, domain=None):
     domain = domain or cls.domain
     user = CommCareUser(username=normalize_username(username, domain),
                         domain=domain)
     user.domain_membership = DomainMembership(domain=domain)
     doc = user._doc
     doc['username.exact'] = doc['username']
     doc['base_username'] = username
     UserESFake.save_doc(doc)
     return user
示例#16
0
 def make_mobile_worker(cls, username, domain=None):
     domain = domain or cls.domain
     user = CommCareUser(username=normalize_username(username, domain),
                         domain=domain)
     user.domain_membership = DomainMembership(domain=domain)
     doc = user._doc
     doc['username.exact'] = doc['username']
     doc['base_username'] = username
     UserESFake.save_doc(doc)
     return user
示例#17
0
        def create_or_update_safe(username, password, uuid, date, registering_phone_id, domain, user_data, **kwargs):
            # check for uuid conflicts, if one exists, respond with the already-created user
            conflicting_user = CommCareUser.get_by_user_id(uuid)

            # we need to check for username conflicts, other issues
            # and make sure we send the appropriate conflict response to the phone
            try:
                username = normalize_username(username, domain)
            except ValidationError:
                raise Exception("Username (%s) is invalid: valid characters include [a-z], "
                                "[0-9], period, underscore, and single quote" % username)

            if conflicting_user:
                # try to update. If there are username conflicts, we have to resolve them
                if conflicting_user.domain != domain:
                    raise Exception("Found a conflicting user in another domain. This is not allowed!")

                saved = False
                to_append = 2
                prefix, suffix = username.split("@")
                while not saved and to_append < MAX_DUPLICATE_USERS:
                    try:
                        conflicting_user.change_username(username)
                        conflicting_user.password = password
                        conflicting_user.date = date
                        conflicting_user.device_id = registering_phone_id
                        conflicting_user.user_data = user_data
                        conflicting_user.save()
                        saved = True
                    except CouchUser.Inconsistent:
                        username = "******" % {
                                     "pref": prefix, "count": to_append,
                                     "suff": suffix}
                        to_append = to_append + 1
                if not saved:
                    raise Exception("There are over 1,000,000 users with that base name in your domain. REALLY?!? REALLY?!?!")
                return (conflicting_user, False)

            try:
                User.objects.get(username=username)
            except User.DoesNotExist:
                # Desired outcome
                pass
            else:
                # Come up with a suitable username
                prefix, suffix = username.split("@")
                username = get_unique_value(User.objects, "username", prefix, sep="", suffix="@%s" % suffix)
            couch_user = cls.create(domain, username, password,
                uuid=uuid,
                device_id=registering_phone_id,
                date=date,
                user_data=user_data
            )
            return (couch_user, True)
示例#18
0
    def post(self, request):
        """View's dispatch method automatically calls this"""

        try:
            workbook = WorkbookJSONReader(request.file)
        except AttributeError:
            return HttpResponseBadRequest("Error processing your Excel (.xlsx) file")

        try:
            data_types = workbook.get_worksheet(title='types')
        except KeyError:
            return HttpResponseBadRequest("Workbook does not have a sheet called 'types'")

        for dt in data_types:
            print "DataType"
            print dt

            for di in workbook.get_worksheet(title=dt['tag']):
                print "DataItem"
                print di

        for dt in data_types:
            data_type = FixtureDataType(
                domain=self.domain,
                name=dt['name'],
                tag=dt['tag'],
                fields=dt['field'],
            )
            data_type.save()
            data_items = workbook.get_worksheet(data_type.tag)
            for di in data_items:
                print di
                data_item = FixtureDataItem(
                    domain=self.domain,
                    data_type_id=data_type.get_id,
                    fields=di['field']
                )
                data_item.save()
                for group_name in di.get('group', []):
                    group = Group.by_name(self.domain, group_name)
                    if group:
                        data_item.add_group(group)
                    else:
                        messages.error(request, "Unknown group: %s" % group_name)
                for raw_username in di.get('user', []):
                    username = normalize_username(raw_username, self.domain)
                    user = CommCareUser.get_by_username(username)
                    if user:
                        data_item.add_user(user)
                    else:
                        messages.error(request, "Unknown user: %s" % raw_username)

        return HttpResponseRedirect(reverse('fixture_view', args=[self.domain]))
示例#19
0
    def setUp(self):
        super().setUp()
        delete_all_users()

        self.domain_obj = create_domain(self.domain)
        enable_usercase(self.domain)

        with trap_extra_setup(ConnectionError):
            self.es = get_es_new()
            initialize_index_and_mapping(self.es, CASE_SEARCH_INDEX_INFO)

        username = normalize_username("mobile_worker_1", self.domain)
        self.mobile_worker = CommCareUser.create(self.domain, username, "123", None, None)
        sync_usercases(self.mobile_worker, self.domain)
示例#20
0
    def set_up_auth_test(self):
        self._set_up_domain()

        try:
            self.user = CommCareUser.create(
                self.domain, username=normalize_username("danny", self.domain), password="******"
            )
        except CommCareUser.Inconsistent:
            pass

        self.app = Application.new_app(self.domain, "My Crazy App")
        self.app.save()

        self.url = reverse(secure_post, args=[self.domain, self.app.get_id])
示例#21
0
 def get_id(type, origin, name):
     """
         i.e.
         get_id('users', 'remote', 'barbara')
     """
     if origin == 'remote':
         return remote_mapping[type][name]
     if (type, origin) == ('users', 'local'):
         username = normalize_username(name, domain)
         user = CommCareUser.get_by_username(username)
         if user:
             return user.user_id
         else:
             raise Exception('Unknown local user "%s"' % username)
     if (type, origin) == ('groups', 'local'):
         return Group.by_name(domain, name).get_id
示例#22
0
 def get_id(type, origin, name):
     """
         i.e.
         get_id('users', 'remote', 'barbara')
     """
     if origin == 'remote':
         return remote_mapping[type][name]
     if (type, origin) == ('users', 'local'):
         username = normalize_username(name, domain)
         user = CommCareUser.get_by_username(username)
         if user:
             return user.user_id
         else:
             raise Exception('Unknown local user "%s"' % username)
     if (type, origin) == ('groups', 'local'):
         return Group.by_name(domain, name).get_id
示例#23
0
    def setUp(self):
        self._set_up_domain()

        try:
            self.user = CommCareUser.create(
                self.domain,
                username=normalize_username('danny', self.domain),
                password='******',
            )
        except CommCareUser.Inconsistent:
            pass

        self.app = Application.new_app(self.domain, 'My Crazy App', '2.0')
        self.app.save()

        self.url = reverse(secure_post, args=[self.domain, self.app.get_id])
示例#24
0
    def _set_up_auth_test(self):
        self._set_up_domain()

        try:
            self.user = CommCareUser.create(
                self.domain,
                username=normalize_username('danny', self.domain),
                password='******',
            )
        except CommCareUser.Inconsistent:
            pass

        self.app = Application.new_app(self.domain, 'My Crazy App')
        self.app.save()

        self.url = reverse(secure_post, args=[self.domain, self.app.get_id])
示例#25
0
    def _run_upload(self, request, workbook):
        group_memoizer = GroupMemoizer(self.domain)

        data_types = workbook.get_worksheet(title='types')

        def _get_or_raise(container, attr, message):
            try:
                return container[attr]
            except KeyError:
                raise Exception(message.format(attr=attr))
        with CouchTransaction() as transaction:
            for dt in data_types:
                err_msg = "Workbook 'types' has no column '{attr}'"
                data_type = FixtureDataType(
                    domain=self.domain,
                    name=_get_or_raise(dt, 'name', err_msg),
                    tag=_get_or_raise(dt, 'tag', err_msg),
                    fields=_get_or_raise(dt, 'field', err_msg),
                )
                transaction.save(data_type)
                data_items = workbook.get_worksheet(data_type.tag)
                for sort_key, di in enumerate(data_items):
                    data_item = FixtureDataItem(
                        domain=self.domain,
                        data_type_id=data_type.get_id,
                        fields=di['field'],
                        sort_key=sort_key
                    )
                    transaction.save(data_item)
                    for group_name in di.get('group', []):
                        group = group_memoizer.by_name(group_name)
                        if group:
                            data_item.add_group(group, transaction=transaction)
                        else:
                            messages.error(request, "Unknown group: %s" % group_name)
                    for raw_username in di.get('user', []):
                        username = normalize_username(raw_username, self.domain)
                        user = CommCareUser.get_by_username(username)
                        if user:
                            data_item.add_user(user)
                        else:
                            messages.error(request, "Unknown user: %s" % raw_username)
            for data_type in transaction.preview_save(cls=FixtureDataType):
                for duplicate in FixtureDataType.by_domain_tag(domain=self.domain, tag=data_type.tag):
                    duplicate.recursive_delete(transaction)
示例#26
0
    def get_exising_users(self):
        usernames_without_ids = set()

        for row in self.all_specs:
            username = row.get('username')
            if row.get('user_id') or not username:
                continue

            try:
                usernames_without_ids.add(normalize_username(username, self.domain))
            except ValidationError:
                pass

        existing_usernames = set()
        for usernames in chunked(usernames_without_ids, 500):
            existing_usernames.update(get_existing_usernames(usernames))

        return existing_usernames
示例#27
0
def _get_latest_enabled_build(domain, username, app_id, profile_id, location_flag_enabled):
    """
    :return: enabled build for the app for a location or profile on basis of feature flag enabled with
    location flag taking precedence
    """
    latest_enabled_build = None
    if location_flag_enabled:
        user = CommCareUser.get_by_username(normalize_username(username, domain))
        user_location_id = user.location_id
        if user_location_id:
            parent_app_id = get_app(domain, app_id).copy_of
            latest_enabled_build = get_latest_app_release_by_location(domain, user_location_id, parent_app_id)
    if not latest_enabled_build:
        # Fall back to the old logic to support migration
        # ToDo: Remove this block once migration is complete
        if profile_id and toggles.RELEASE_BUILDS_PER_PROFILE.enabled(domain):
            latest_enabled_build = get_latest_enabled_build_for_profile(domain, profile_id)
    return latest_enabled_build
示例#28
0
def _get_latest_enabled_build(domain, username, app_id, profile_id, location_flag_enabled):
    """
    :return: enabled build for the app for a location or profile on basis of feature flag enabled with
    location flag taking precedence
    """
    latest_enabled_build = None
    if location_flag_enabled:
        user = CommCareUser.get_by_username(normalize_username(username, domain))
        user_location_id = user.location_id
        if user_location_id:
            parent_app_id = get_app(domain, app_id).copy_of
            latest_enabled_build = get_latest_app_release_by_location(domain, user_location_id, parent_app_id)
    if not latest_enabled_build:
        # Fall back to the old logic to support migration
        # ToDo: Remove this block once migration is complete
        if profile_id and toggles.RELEASE_BUILDS_PER_PROFILE.enabled(domain):
            latest_enabled_build = get_latest_enabled_build_for_profile(domain, profile_id)
    return latest_enabled_build
示例#29
0
    def setUpClass(cls):
        cls.report_id = '7b97e8b53d00d43ca126b10093215a9d'
        cls.report_config_uuid = 'a98c812873986df34fd1b4ceb45e6164ae9cc664'
        cls.domain = 'report-filter-test-domain'
        cls.user = CommCareUser(
            username=normalize_username('ralph', cls.domain),
            domain=cls.domain,
            language='en',
        )
        update_toggle_cache(MOBILE_UCR.slug, cls.domain, True, NAMESPACE_DOMAIN)

        report_configuration = cls.make_report_config(cls.domain, cls.report_id)
        cls.report_configs_by_id = {
            cls.report_id: report_configuration
        }
        cls.app = Application.new_app(cls.domain, "Report Filter Test App", APP_V2)
        module = cls.app.add_module(ReportModule.new_module("Report Module", 'en'))
        module.report_configs.append(
            ReportAppConfig(
                report_id=cls.report_id,
                header={},
                description="",
                graph_configs={
                    '7451243209119342931': ReportGraphConfig(
                        series_configs={'count': {}}
                    )
                },
                filters={'computed_owner_name_40cc88a0_1': MobileSelectFilter()},
                uuid=cls.report_config_uuid,
            )
        )
        with mock_report_configurations(cls.report_configs_by_id):
            cls.suite = cls.app.create_suite()
        cls.data = [
            {'color_94ec39e6': 'red', 'count': 2, 'computed_owner_name_40cc88a0': 'cory'},
            {'color_94ec39e6': 'black', 'count': 1, 'computed_owner_name_40cc88a0': 'ctsims'},
            {'color_94ec39e6': 'red', 'count': 3, 'computed_owner_name_40cc88a0': 'daniel'},
        ]
        with mock_report_data(cls.data):
            with mock_report_configuration_get(cls.report_configs_by_id):
                with mock.patch('corehq.apps.app_manager.fixtures.mobile_ucr.get_apps_in_domain',
                                lambda domain, include_remote: [cls.app]):
                    fixture, = report_fixture_generator(cls.user, '2.0')
        cls.fixture = ElementTree.tostring(fixture)
示例#30
0
def create_or_update_locations(domain, location_specs, log):
    location_cache = LocationCache()
    users = {}
    for row in location_specs:
        username = normalize_username(row.get('username'), domain)
        location_code = row.get('location-sms-code')
        if username in users:
            user_mapping = users[username]
        else:
            user_mapping = UserLocMapping(username, domain, location_cache)
            users[username] = user_mapping

        if row.get('remove') == 'y':
            user_mapping.to_remove.add(location_code)
        else:
            user_mapping.to_add.add(location_code)

    for username, mapping in users.iteritems():
        mapping.save()
示例#31
0
    def setUp(self):
        self.domain = 'my-crazy-domain'
        create_domain(self.domain)

        try:
            self.user = CommCareUser.create(
                self.domain,
                username=normalize_username('danny', self.domain),
                password='******',
            )
        except CommCareUser.Inconsistent:
            pass

        self.app = Application.new_app(self.domain, 'My Crazy App', '2.0')
        self.app.save()

        self.file_path = os.path.join(
            os.path.dirname(__file__), "data", 'bare_form.xml'
        )
        self.url = reverse(secure_post, args=[self.domain, self.app.get_id])
示例#32
0
    def _run_upload(self, request, workbook):
        group_memoizer = GroupMemoizer(self.domain)

        data_types = workbook.get_worksheet(title='types')

        with CouchTransaction() as transaction:
            for dt in data_types:
                data_type = FixtureDataType(
                    domain=self.domain,
                    name=dt['name'],
                    tag=dt['tag'],
                    fields=dt['field'],
                )
                transaction.save(data_type)
                data_items = workbook.get_worksheet(data_type.tag)
                for di in data_items:
                    data_item = FixtureDataItem(
                        domain=self.domain,
                        data_type_id=data_type.get_id,
                        fields=di['field']
                    )
                    transaction.save(data_item)
                    for group_name in di.get('group', []):
                        group = group_memoizer.by_name(group_name)
                        if group:
                            data_item.add_group(group, transaction=transaction)
                        else:
                            messages.error(request, "Unknown group: %s" % group_name)
                    for raw_username in di.get('user', []):
                        username = normalize_username(raw_username, self.domain)
                        user = CommCareUser.get_by_username(username)
                        if user:
                            data_item.add_user(user)
                        else:
                            messages.error(request, "Unknown user: %s" % raw_username)
            for data_type in transaction.preview_save(cls=FixtureDataType):
                for duplicate in FixtureDataType.by_domain_tag(domain=self.domain, tag=data_type.tag):
                    duplicate.recursive_delete(transaction)
示例#33
0
def create_or_update_locations(domain, location_specs, log):
    location_cache = LocationCache()
    users = {}
    for row in location_specs:
        username = row.get('username')
        try:
            username = normalize_username(username, domain)
        except ValidationError:
            log['errors'].append("Username must be a valid email address: %s" % username)
        else:
            location_code = unicode(row.get('location-sms-code'))
            if username in users:
                user_mapping = users[username]
            else:
                user_mapping = UserLocMapping(username, domain, location_cache)
                users[username] = user_mapping

            if row.get('remove') == 'y':
                user_mapping.to_remove.add(location_code)
            else:
                user_mapping.to_add.add(location_code)

    for username, mapping in users.iteritems():
        mapping.save()
示例#34
0
def create_or_update_users_and_groups(domain, user_specs, group_specs):
    ret = {"errors": [], "rows": []}
    group_memoizer = create_or_update_groups(domain, group_specs, log=ret)
    usernames = set()
    user_ids = set()

    allowed_groups = set(group_memoizer.groups)
    allowed_group_names = [group.name for group in allowed_groups]

    try:
        for row in user_specs:
            data, group_names, language, name, password, phone_number, user_id, username = (
                row.get(k) for k in sorted(allowed_headers))
            password = unicode(password)
            group_names = group_names or []
            try:
                username = normalize_username(username, domain)
            except TypeError:
                username = None
            except ValidationError:
                ret['rows'].append({
                    'username':
                    username,
                    'row':
                    row,
                    'flag':
                    _('username cannot contain spaces or symbols'),
                })
                continue
            status_row = {
                'username': raw_username(username) if username else None,
                'row': row,
            }
            if username in usernames or user_id in user_ids:
                status_row['flag'] = 'repeat'
            elif not username and not user_id:
                status_row['flag'] = 'missing-data'
            else:
                try:
                    if username:
                        usernames.add(username)
                    if user_id:
                        user_ids.add(user_id)
                    if user_id:
                        user = CommCareUser.get_by_user_id(user_id, domain)
                    else:
                        user = CommCareUser.get_by_username(username)
                    if user:
                        if user.domain != domain:
                            raise UserUploadError(
                                _('User with username %(username)r is somehow in domain %(domain)r'
                                  ) % {
                                      'username': user.username,
                                      'domain': user.domain
                                  })
                        if username and user.username != username:
                            user.change_username(username)
                        if password:
                            user.set_password(password)
                        status_row['flag'] = 'updated'
                    else:
                        if not password:
                            raise UserUploadError(
                                _("Cannot create a new user with a blank password"
                                  ))
                        user = CommCareUser.create(domain,
                                                   username,
                                                   password,
                                                   uuid=user_id or '')
                        status_row['flag'] = 'created'
                    if phone_number:
                        user.add_phone_number(_fmt_phone(phone_number),
                                              default=True)
                    if name:
                        user.set_full_name(name)
                    if data:
                        user.user_data.update(data)
                    if language:
                        user.language = language
                    user.save()
                    if password:
                        # Without this line, digest auth doesn't work.
                        # With this line, digest auth works.
                        # Other than that, I'm not sure what's going on
                        user.get_django_user().check_password(password)

                    for group_id in Group.by_user(user, wrap=False):
                        group = group_memoizer.get(group_id)
                        if group.name not in group_names:
                            group.remove_user(user, save=False)

                    for group_name in group_names:
                        if group_name not in allowed_group_names:
                            raise UserUploadError(
                                _("Can't add to group '%s' "
                                  "(try adding it to your spreadsheet)") %
                                group_name)
                        group_memoizer.by_name(group_name).add_user(user,
                                                                    save=False)

                except UserUploadError as e:
                    status_row['flag'] = '%s' % e

            ret["rows"].append(status_row)
    finally:
        group_memoizer.save_all()

    return ret
示例#35
0
def create_or_update_users_and_groups(upload_domain,
                                      user_specs,
                                      upload_user,
                                      group_memoizer=None,
                                      update_progress=None):
    domain_info_by_domain = {}

    ret = {"errors": [], "rows": []}

    current = 0

    try:
        for row in user_specs:
            if update_progress:
                update_progress(current)
                current += 1
            log_user_create = False
            log_role_update = False

            username = row.get('username')
            domain = row.get('domain') or upload_domain
            username = normalize_username(str(username),
                                          domain) if username else None
            status_row = {
                'username': username,
                'row': row,
            }

            domain_info = get_domain_info(domain, upload_domain, user_specs,
                                          domain_info_by_domain,
                                          group_memoizer)

            try:
                for validator in domain_info.validators:
                    validator(row)
            except UserUploadError as e:
                status_row['flag'] = str(e)
                ret['rows'].append(status_row)
                continue

            data = row.get('data', {})
            email = row.get('email')
            group_names = list(map(str, row.get('group') or []))
            language = row.get('language')
            name = row.get('name')
            password = row.get('password')
            phone_number = row.get('phone-number')
            uncategorized_data = row.get('uncategorized_data', {})
            user_id = row.get('user_id')
            location_codes = row.get('location_code',
                                     []) if 'location_code' in row else None
            location_codes = format_location_codes(location_codes)
            role = row.get('role', None)
            profile = row.get('user_profile', None)
            web_user = row.get('web_user')

            try:
                password = str(password) if password else None

                is_active = spec_value_to_boolean_or_none(row, 'is_active')
                is_account_confirmed = spec_value_to_boolean_or_none(
                    row, 'is_account_confirmed')
                send_account_confirmation_email = spec_value_to_boolean_or_none(
                    row, 'send_confirmation_email')
                remove_web_user = spec_value_to_boolean_or_none(
                    row, 'remove_web_user')

                if user_id:
                    user = CommCareUser.get_by_user_id(user_id, domain)
                    if not user:
                        raise UserUploadError(
                            _("User with ID '{user_id}' not found").format(
                                user_id=user_id, domain=domain))
                    check_changing_username(user, username)

                    # note: explicitly not including "None" here because that's the default value if not set.
                    # False means it was set explicitly to that value
                    if is_account_confirmed is False and not web_user:
                        raise UserUploadError(
                            _("You can only set 'Is Account Confirmed' to 'False' on a new User."
                              ))

                    if is_password(password):
                        user.set_password(password)
                        # overwrite password in results so we do not save it to the db
                        status_row['row']['password'] = '******'
                    status_row['flag'] = 'updated'
                else:
                    kwargs = {}
                    if is_account_confirmed is not None and not web_user:
                        kwargs['is_account_confirmed'] = is_account_confirmed
                    user = CommCareUser.create(
                        domain,
                        username,
                        password,
                        created_by=upload_user,
                        created_via=USER_CHANGE_VIA_BULK_IMPORTER,
                        commit=False,
                        **kwargs)
                    log_user_create = True
                    status_row['flag'] = 'created'

                if phone_number:
                    user.add_phone_number(_fmt_phone(phone_number),
                                          default=True)
                if name:
                    user.set_full_name(str(name))

                # Add in existing data. Don't use metadata - we don't want to add profile-controlled fields.
                for key, value in user.user_data.items():
                    if key not in data:
                        data[key] = value
                if profile:
                    profile_obj = domain_info.profiles_by_name[profile]
                    data[PROFILE_SLUG] = profile_obj.id
                    for key in profile_obj.fields.keys():
                        user.pop_metadata(key)
                try:
                    user.update_metadata(data)
                except ValueError as e:
                    raise UserUploadError(str(e))
                if uncategorized_data:
                    user.update_metadata(uncategorized_data)

                # Clear blank user data so that it can be purged by remove_unused_custom_fields_from_users_task
                for key in dict(data, **uncategorized_data):
                    value = user.metadata[key]
                    if value is None or value == '':
                        user.pop_metadata(key)

                if language:
                    user.language = language
                if email:
                    user.email = email.lower()
                if is_active is not None:
                    user.is_active = is_active

                if domain_info.can_assign_locations and location_codes is not None:
                    # Do this here so that we validate the location code before we
                    # save any other information to the user, this way either all of
                    # the user's information is updated, or none of it

                    # Do not update location info if the column is not included at all
                    location_ids = find_location_id(location_codes,
                                                    domain_info.location_cache)
                    locations_updated, primary_loc_removed = check_modified_user_loc(
                        location_ids, user.location_id,
                        user.assigned_location_ids)
                    if primary_loc_removed:
                        user.unset_location(commit=False)
                    if locations_updated:
                        user.reset_locations(location_ids, commit=False)

                if role:
                    role_qualified_id = domain_info.roles_by_name[
                        role].get_qualified_id()
                    user_current_role = user.get_role(domain=domain)
                    log_role_update = not (
                        user_current_role
                        and user_current_role.get_qualified_id()
                        == role_qualified_id)
                    if log_role_update:
                        user.set_role(domain, role_qualified_id)

                if web_user:
                    user.update_metadata({'login_as_user': web_user})

                user.save()
                if log_user_create:
                    user.log_user_create(upload_user,
                                         USER_CHANGE_VIA_BULK_IMPORTER)
                if log_role_update:
                    log_user_role_update(domain, user, upload_user,
                                         USER_CHANGE_VIA_BULK_IMPORTER)
                if web_user:
                    check_can_upload_web_users(upload_user)
                    current_user = CouchUser.get_by_username(web_user)
                    if remove_web_user:
                        remove_web_user_from_domain(domain, current_user,
                                                    username, upload_user)
                    else:
                        check_user_role(username, role)
                        if not current_user and is_account_confirmed:
                            raise UserUploadError(
                                _("You can only set 'Is Account Confirmed' to 'True' on an existing Web User. {web_user} is a new username."
                                  ).format(web_user=web_user))
                        if current_user and not current_user.is_member_of(
                                domain) and is_account_confirmed:
                            current_user.add_as_web_user(
                                domain,
                                role=role_qualified_id,
                                location_id=user.location_id)

                        elif not current_user or not current_user.is_member_of(
                                domain):
                            create_or_update_web_user_invite(
                                web_user,
                                domain,
                                role_qualified_id,
                                upload_user,
                                user.location_id,
                                send_email=send_account_confirmation_email)

                        elif current_user.is_member_of(domain):
                            # edit existing user in the domain
                            current_user.set_role(domain, role_qualified_id)
                            if location_codes is not None:
                                if user.location_id:
                                    current_user.set_location(
                                        domain, user.location_id)
                                else:
                                    current_user.unset_location(domain)
                            current_user.save()

                if send_account_confirmation_email and not web_user:
                    send_account_confirmation_if_necessary(user)

                if is_password(password):
                    # Without this line, digest auth doesn't work.
                    # With this line, digest auth works.
                    # Other than that, I'm not sure what's going on
                    # Passing use_primary_db=True because of https://dimagi-dev.atlassian.net/browse/ICDS-465
                    user.get_django_user(
                        use_primary_db=True).check_password(password)

                for group in domain_info.group_memoizer.by_user_id(
                        user.user_id):
                    if group.name not in group_names:
                        group.remove_user(user)

                for group_name in group_names:
                    domain_info.group_memoizer.by_name(group_name).add_user(
                        user, save=False)

            except (UserUploadError, CouchUser.Inconsistent) as e:
                status_row['flag'] = str(e)

            ret["rows"].append(status_row)
    finally:
        try:
            for domain_info in domain_info_by_domain.values():
                domain_info.group_memoizer.save_all()
        except BulkSaveError as e:
            _error_message = (
                "Oops! We were not able to save some of your group changes. "
                "Please make sure no one else is editing your groups "
                "and try again.")
            logging.exception(('BulkSaveError saving groups. '
                               'User saw error message "%s". Errors: %s') %
                              (_error_message, e.errors))
            ret['errors'].append(_error_message)

    return ret
示例#36
0
def run_upload_api(request, domain, workbook):
    return_val = {
        "unknown_groups": [],
        "unknown_users": [],
        "number_of_fixtures": 0,
    }
    group_memoizer = GroupMemoizer(domain)

    data_types = workbook.get_worksheet(title='types')

    def _get_or_raise(container, attr):
        try:
            return container[attr]
        except KeyError:
            raise Exception("Workbook 'types' has no column '{attr}'".format(attr=attr))
   
    number_of_fixtures = -1
    with CouchTransaction() as transaction:
        for number_of_fixtures, dt in enumerate(data_types):
            tag = _get_or_raise(dt, 'tag')
            data_type_results = FixtureDataType.by_domain_tag(domain, tag)
            if len(data_type_results) == 0:
                data_type = FixtureDataType(
                    domain=domain,
                    name=_get_or_raise(dt, 'name'),
                    tag=_get_or_raise(dt, 'tag'),
                    fields=_get_or_raise(dt, 'field'),
                )
                transaction.save(data_type)
            else:
                for x in data_type_results:
                    data_type = x

            data_items = workbook.get_worksheet(data_type.tag)
            for sort_key, di in enumerate(data_items):
                new_data_item = FixtureDataItem(
                    domain=domain,
                    data_type_id=data_type.get_id,
                    fields=di['field'],
                    sort_key=sort_key
                )
                try:
                    old_data_item = FixtureDataItem.get(di['UID'])
                    assert old_data_item.domain == domain
                    assert old_data_item.doc_type == FixtureDataItem._doc_type
                    assert old_data_item.data_type_id == data_type.get_id
                    if di.get(DELETE_HEADER) in ("Y", "y"):
                        old_data_item.recursive_delete(transaction)
                        continue
                    old_data_item.fields = di['field']
                    transaction.save(old_data_item)
                except (AttributeError, KeyError, ResourceNotFound,
                        AssertionError):
                    old_data_item = new_data_item
                    transaction.save(old_data_item)

                old_groups = old_data_item.get_groups()
                for group in old_groups:
                    old_data_item.remove_group(group)
                old_users = old_data_item.get_users()
                for user in old_users:
                    old_data_item.remove_user(user)

                for group_name in di.get('group', []):
                        group = group_memoizer.by_name(group_name)
                        if group:
                            old_data_item.add_group(group, transaction=transaction)
                        else:
                            messages.error(request, "Unknown group: %s" % group_name)

                for raw_username in di.get('user', []):
                        username = normalize_username(raw_username, domain)
                        user = CommCareUser.get_by_username(username)
                        if user:
                            old_data_item.add_user(user)
                        else:
                            messages.error(request, "Unknown user: %s" % raw_username)

    return_val["number_of_fixtures"] = number_of_fixtures + 1
    return return_val
示例#37
0
def run_upload(domain, workbook, replace=False, task=None):
    return_val = FixtureUploadResult()
    group_memoizer = GroupMemoizer(domain)
    get_location = get_memoized_location(domain)

    def diff_lists(list_a, list_b):
        set_a = set(list_a)
        set_b = set(list_b)
        not_in_b = set_a.difference(set_b)
        not_in_a = set_a.difference(set_a)
        return list(not_in_a), list(not_in_b)

    with CouchTransaction() as transaction:
        type_sheets = workbook.get_all_type_sheets()
        total_tables = len(type_sheets)
        return_val.number_of_fixtures = total_tables

        def _update_progress(table_count, item_count, items_in_table):
            if task:
                processed = table_count * 10 + (10. * item_count /
                                                items_in_table)
                DownloadBase.set_progress(task, processed, 10 * total_tables)

        for table_number, table_def in enumerate(type_sheets):
            tag = table_def.table_id
            new_data_type = FixtureDataType(
                domain=domain,
                is_global=table_def.is_global,
                tag=table_def.table_id,
                fields=table_def.fields,
                item_attributes=table_def.item_attributes)
            try:
                tagged_fdt = FixtureDataType.fixture_tag_exists(domain, tag)
                if tagged_fdt:
                    data_type = tagged_fdt
                # support old usage with 'UID'
                elif table_def.uid:
                    data_type = FixtureDataType.get(table_def.uid)
                else:
                    data_type = new_data_type

                if replace and data_type != new_data_type:
                    data_type.recursive_delete(transaction)
                    data_type = new_data_type

                data_type.fields = table_def.fields
                data_type.item_attributes = table_def.item_attributes
                data_type.is_global = table_def.is_global
                assert data_type.doc_type == FixtureDataType._doc_type
                if data_type.domain != domain:
                    data_type = new_data_type
                    return_val.errors.append(
                        _("'%(UID)s' is not a valid UID. But the new type is created."
                          ) % {'UID': table_def.uid})
                if table_def.delete:
                    data_type.recursive_delete(transaction)
                    continue
            except (ResourceNotFound, KeyError) as e:
                data_type = new_data_type
            transaction.save(data_type)

            data_items = list(workbook.get_data_sheet(data_type))
            items_in_table = len(data_items)
            for sort_key, di in enumerate(data_items):
                _update_progress(table_number, sort_key, items_in_table)
                # Check that type definitions in 'types' sheet vs corresponding columns in the item-sheet MATCH
                item_fields_list = di['field'].keys() if 'field' in di else []
                not_in_sheet, not_in_types = diff_lists(
                    item_fields_list, data_type.fields_without_attributes)
                if len(not_in_sheet) > 0:
                    error_message = _(
                        FAILURE_MESSAGES["has_no_field_column"]).format(
                            tag=tag, field=not_in_sheet[0])
                    raise ExcelMalformatException(error_message)
                if len(not_in_types) > 0:
                    error_message = _(
                        FAILURE_MESSAGES["has_extra_column"]).format(
                            tag=tag, field=not_in_types[0])
                    raise ExcelMalformatException(error_message)

                # check that this item has all the properties listed in its 'types' definition
                item_attributes_list = di['property'].keys(
                ) if 'property' in di else []
                not_in_sheet, not_in_types = diff_lists(
                    item_attributes_list, data_type.item_attributes)
                if len(not_in_sheet) > 0:
                    error_message = _(
                        FAILURE_MESSAGES["has_no_field_column"]).format(
                            tag=tag, field=not_in_sheet[0])
                    raise ExcelMalformatException(error_message)
                if len(not_in_types) > 0:
                    error_message = _(
                        FAILURE_MESSAGES["has_extra_column"]).format(
                            tag=tag, field=not_in_types[0])
                    raise ExcelMalformatException(error_message)

                # check that properties in 'types' sheet vs item-sheet MATCH
                for field in data_type.fields:
                    if len(field.properties) > 0:
                        sheet_props = di.get(field.field_name, {})
                        sheet_props_list = sheet_props.keys()
                        type_props = field.properties
                        not_in_sheet, not_in_types = diff_lists(
                            sheet_props_list, type_props)
                        if len(not_in_sheet) > 0:
                            error_message = _(
                                FAILURE_MESSAGES["sheet_has_no_property"]
                            ).format(tag=tag,
                                     property=not_in_sheet[0],
                                     field=field.field_name)
                            raise ExcelMalformatException(error_message)
                        if len(not_in_types) > 0:
                            error_message = _(
                                FAILURE_MESSAGES["sheet_has_extra_property"]
                            ).format(tag=tag,
                                     property=not_in_types[0],
                                     field=field.field_name)
                            raise ExcelMalformatException(error_message)
                        # check that fields with properties are numbered
                        if type(di['field'][field.field_name]) != list:
                            error_message = _(
                                FAILURE_MESSAGES["invalid_field_with_property"]
                            ).format(field=field.field_name)
                            raise ExcelMalformatException(error_message)
                        field_prop_len = len(di['field'][field.field_name])
                        for prop in sheet_props:
                            if type(sheet_props[prop]) != list:
                                error_message = _(
                                    FAILURE_MESSAGES["invalid_property"]
                                ).format(field=field.field_name, prop=prop)
                                raise ExcelMalformatException(error_message)
                            if len(sheet_props[prop]) != field_prop_len:
                                error_message = _(FAILURE_MESSAGES[
                                    "wrong_field_property_combos"]).format(
                                        field=field.field_name, prop=prop)
                                raise ExcelMalformatException(error_message)

                # excel format check should have been covered by this line. Can make assumptions about data now
                type_fields = data_type.fields
                item_fields = {}
                for field in type_fields:
                    # if field doesn't have properties
                    if len(field.properties) == 0:
                        item_fields[field.field_name] = FieldList(field_list=[
                            FixtureItemField(
                                # using unicode here, to cast ints, and multi-language strings
                                field_value=unicode(di['field'][
                                    field.field_name]),
                                properties={})
                        ])
                    else:
                        field_list = []
                        field_prop_combos = di['field'][field.field_name]
                        prop_combo_len = len(field_prop_combos)
                        prop_dict = di[field.field_name]
                        for x in range(0, prop_combo_len):
                            fix_item_field = FixtureItemField(
                                field_value=unicode(field_prop_combos[x]),
                                properties={
                                    prop: unicode(prop_dict[prop][x])
                                    for prop in prop_dict
                                })
                            field_list.append(fix_item_field)
                        item_fields[field.field_name] = FieldList(
                            field_list=field_list)

                item_attributes = di.get('property', {})
                new_data_item = FixtureDataItem(
                    domain=domain,
                    data_type_id=data_type.get_id,
                    fields=item_fields,
                    item_attributes=item_attributes,
                    sort_key=sort_key)
                try:
                    if di['UID'] and not replace:
                        old_data_item = FixtureDataItem.get(di['UID'])
                    else:
                        old_data_item = new_data_item
                        pass
                    old_data_item.fields = item_fields
                    old_data_item.item_attributes = item_attributes
                    if old_data_item.domain != domain or not old_data_item.data_type_id == data_type.get_id:
                        old_data_item = new_data_item
                        return_val.errors.append(
                            _("'%(UID)s' is not a valid UID. But the new item is created."
                              ) % {'UID': di['UID']})
                    assert old_data_item.doc_type == FixtureDataItem._doc_type
                    if di[DELETE_HEADER] == "Y" or di[DELETE_HEADER] == "y":
                        old_data_item.recursive_delete(transaction)
                        continue
                except (ResourceNotFound, KeyError):
                    old_data_item = new_data_item
                transaction.save(old_data_item)

                old_groups = old_data_item.groups
                for group in old_groups:
                    old_data_item.remove_group(group)
                old_users = old_data_item.users
                for user in old_users:
                    old_data_item.remove_user(user)
                old_locations = old_data_item.locations
                for location in old_locations:
                    old_data_item.remove_location(location)

                for group_name in di.get('group', []):
                    group = group_memoizer.by_name(group_name)
                    if group:
                        old_data_item.add_group(group, transaction=transaction)
                    else:
                        return_val.errors.append(
                            _("Unknown group: '%(name)s'. But the row is successfully added"
                              ) % {'name': group_name})

                for raw_username in di.get('user', []):
                    try:
                        username = normalize_username(str(raw_username),
                                                      domain)
                    except ValidationError:
                        return_val.errors.append(
                            _("Invalid username: '******'. Row is not added")
                            % {'name': raw_username})
                        continue
                    user = CommCareUser.get_by_username(username)
                    if user:
                        old_data_item.add_user(user)
                    else:
                        return_val.errors.append(
                            _("Unknown user: '******'. But the row is successfully added"
                              ) % {'name': raw_username})

                for name in di.get('location', []):
                    location_cache = get_location(name)
                    if location_cache.is_error:
                        return_val.errors.append(location_cache.message)
                    else:
                        old_data_item.add_location(location_cache.location,
                                                   transaction=transaction)

    return return_val
示例#38
0
def _run_fixture_upload(domain, workbook, replace=False, task=None):
    from corehq.apps.users.bulkupload import GroupMemoizer
    return_val = FixtureUploadResult()
    group_memoizer = GroupMemoizer(domain)
    get_location = get_memoized_location_getter(domain)
    data_types = []

    with CouchTransaction() as transaction:
        type_sheets = workbook.get_all_type_sheets()
        total_tables = len(type_sheets)
        return_val.number_of_fixtures = total_tables

        def _update_progress(table_count, item_count, items_in_table):
            if task:
                processed = table_count * 10 + (10 * item_count / items_in_table)
                DownloadBase.set_progress(task, processed, 10 * total_tables)

        for table_number, table_def in enumerate(type_sheets):
            tag = table_def.table_id
            new_data_type = FixtureDataType(
                domain=domain,
                is_global=table_def.is_global,
                tag=tag,
                fields=table_def.fields,
                item_attributes=table_def.item_attributes
            )
            try:
                tagged_fdt = FixtureDataType.fixture_tag_exists(domain, tag)
                if tagged_fdt:
                    data_type = tagged_fdt
                # support old usage with 'UID'
                elif table_def.uid:
                    data_type = FixtureDataType.get(table_def.uid)
                else:
                    data_type = new_data_type

                if replace and data_type != new_data_type:
                    data_type.recursive_delete(transaction)
                    data_type = new_data_type

                data_type.fields = table_def.fields
                data_type.item_attributes = table_def.item_attributes
                data_type.is_global = table_def.is_global
                assert data_type.doc_type == FixtureDataType._doc_type
                if data_type.domain != domain:
                    data_type = new_data_type
                    return_val.errors.append(
                        _("'%(UID)s' is not a valid UID. But the new type is created.")
                        % {'UID': table_def.uid}
                    )
                if table_def.delete:
                    data_type.recursive_delete(transaction)
                    continue
            except (ResourceNotFound, KeyError):
                data_type = new_data_type
            transaction.save(data_type)
            data_types.append(data_type)
            data_items = list(workbook.get_data_sheet(data_type.tag))
            items_in_table = len(data_items)
            for sort_key, di in enumerate(data_items):
                _update_progress(table_number, sort_key, items_in_table)
                type_fields = data_type.fields
                item_fields = {}
                for field in type_fields:
                    # if field doesn't have properties
                    if len(field.properties) == 0:
                        item_fields[field.field_name] = FieldList(
                            field_list=[FixtureItemField(
                                # using unicode here, to cast ints, and multi-language strings
                                field_value=six.text_type(di['field'][field.field_name]),
                                properties={}
                            )]
                        )
                    else:
                        field_list = []
                        field_prop_combos = di['field'][field.field_name]
                        prop_combo_len = len(field_prop_combos)
                        prop_dict = di[field.field_name]
                        for x in range(0, prop_combo_len):
                            fix_item_field = FixtureItemField(
                                field_value=six.text_type(field_prop_combos[x]),
                                properties={prop: six.text_type(prop_dict[prop][x]) for prop in prop_dict}
                            )
                            field_list.append(fix_item_field)
                        item_fields[field.field_name] = FieldList(
                            field_list=field_list
                        )

                item_attributes = di.get('property', {})
                new_data_item = FixtureDataItem(
                    domain=domain,
                    data_type_id=data_type.get_id,
                    fields=item_fields,
                    item_attributes=item_attributes,
                    sort_key=sort_key
                )
                try:
                    if di['UID'] and not replace:
                        old_data_item = FixtureDataItem.get(di['UID'])
                    else:
                        old_data_item = new_data_item
                        pass
                    old_data_item.fields = item_fields
                    old_data_item.item_attributes = item_attributes
                    if old_data_item.domain != domain \
                            or not old_data_item.data_type_id == data_type.get_id:
                        old_data_item = new_data_item
                        return_val.errors.append(
                            _("'%(UID)s' is not a valid UID. But the new item is created.")
                            % {'UID': di['UID']}
                        )
                    assert old_data_item.doc_type == FixtureDataItem._doc_type
                    if di[DELETE_HEADER] == "Y" or di[DELETE_HEADER] == "y":
                        old_data_item.recursive_delete(transaction)
                        continue
                except (ResourceNotFound, KeyError):
                    old_data_item = new_data_item
                transaction.save(old_data_item)

                old_groups = old_data_item.groups
                for group in old_groups:
                    old_data_item.remove_group(group)
                old_users = old_data_item.users
                for user in old_users:
                    old_data_item.remove_user(user)
                old_locations = old_data_item.locations
                for location in old_locations:
                    old_data_item.remove_location(location)

                for group_name in di.get('group', []):
                    group = group_memoizer.by_name(group_name)
                    if group:
                        old_data_item.add_group(group, transaction=transaction)
                    else:
                        return_val.errors.append(
                            _("Unknown group: '%(name)s'. But the row is successfully added")
                            % {'name': group_name}
                        )

                for raw_username in di.get('user', []):
                    try:
                        username = normalize_username(str(raw_username), domain)
                    except ValidationError:
                        return_val.errors.append(
                            _("Invalid username: '******'. Row is not added")
                            % {'name': raw_username}
                        )
                        continue
                    user = CommCareUser.get_by_username(username)
                    if user:
                        old_data_item.add_user(user)
                    else:
                        return_val.errors.append(
                            _("Unknown user: '******'. But the row is successfully added")
                            % {'name': raw_username}
                        )

                for name in di.get('location', []):
                    location_cache = get_location(name)
                    if location_cache.is_error:
                        return_val.errors.append(location_cache.message)
                    else:
                        old_data_item.add_location(location_cache.location,
                                                   transaction=transaction)

    clear_fixture_quickcache(domain, data_types)
    clear_fixture_cache(domain)
    return return_val
示例#39
0
def run_upload(request, domain, workbook, replace=False):
    return_val = {
        "unknown_groups": [], 
        "unknown_users": [], 
        "number_of_fixtures": 0,
    }
    failure_messages = {
        "has_no_column": "Workbook 'types' has no column '{column_name}'.",
        "has_no_field_column": "Excel-sheet '{tag}' does not contain the column '{field}' "
                               "as specified in its 'types' definition",
        "has_extra_column": "Excel-sheet '{tag}' has an extra column" + 
                            "'{field}' that's not defined in its 'types' definition",
        "wrong_property_syntax": "Properties should be specified as 'field 1: property 1'. In 'types' sheet, " +
                            "'{prop_key}' for field '{field}' is not correctly formatted",
        "sheet_has_no_property": "Excel-sheet '{tag}' does not contain property " +
                            "'{property}' of the field '{field}' as specified in its 'types' definition",
        "sheet_has_extra_property": "Excel-sheet '{tag}'' has an extra property " +
                            "'{property}' for the field '{field}' that's not defined in its 'types' definition. Re-check the formatting", 
        "invalid_field_with_property": "Fields with attributes should be numbered as 'field: {field} integer",
        "invalid_property": "Attribute should be written as '{field}: {prop} interger'",
        "wrong_field_property_combos": "Number of values for field '{field}' and attribute '{prop}' should be same",
        "replace_with_UID": "Rows shouldn't contain UIDs while using replace option. Excel sheet '{tag}' contains UID in a row."
    }

    group_memoizer = GroupMemoizer(domain)

    data_types = workbook.get_worksheet(title='types')

    def _get_or_raise(container, attr):
        try:
            return container[attr]
        except KeyError:
            raise ExcelMalformatException(_(failure_messages["has_no_column"].format(column_name=attr)))

    def diff_lists(list_a, list_b):
        set_a = set(list_a)
        set_b = set(list_b)
        not_in_b = set_a.difference(set_b)
        not_in_a = set_a.difference(set_a)
        return list(not_in_a), list(not_in_b)
   
    number_of_fixtures = -1
    with CouchTransaction() as transaction:
        fixtures_tags = []
        type_sheets = []
        for number_of_fixtures, dt in enumerate(data_types):
            try:
                tag = _get_or_raise(dt, 'table_id')
            except ExcelMalformatException:
                tag = _get_or_raise(dt, 'tag')
            if tag in fixtures_tags:
                error_message = "Upload Failed: Lookup-tables should have unique 'table_id'. There are two rows with table_id '{tag}' in 'types' sheet."
                raise DuplicateFixtureTagException(_(error_message.format(tag=tag)))
            fixtures_tags.append(tag)
            type_sheets.append(dt)
        for number_of_fixtures, dt in enumerate(type_sheets):
            try:
                tag = _get_or_raise(dt, 'table_id')
            except ExcelMalformatException:
                messages.info(request, _("Excel-header 'tag' is renamed as 'table_id' and 'name' header is no longer needed."))
                tag = _get_or_raise(dt, 'tag')

            type_definition_fields = _get_or_raise(dt, 'field')
            type_fields_with_properties = []
            for count, field in enumerate(type_definition_fields):
                prop_key = "field " + str(count + 1)
                if dt.has_key(prop_key):
                    try:
                        property_list = dt[prop_key]["property"]
                    except KeyError:
                        error_message = failure_messages["wrong_property_syntax"].format(
                            prop_key=prop_key,
                            field=field
                        )
                        raise ExcelMalformatException(_(error_message))
                else:
                    property_list = []
                field_with_prop = FixtureTypeField(
                    field_name=field,
                    properties=property_list
                )
                type_fields_with_properties.append(field_with_prop)

            new_data_type = FixtureDataType(
                domain=domain,
                is_global=dt.get('is_global', False),
                tag=tag,
                fields=type_fields_with_properties,
            )
            try:
                tagged_fdt = FixtureDataType.fixture_tag_exists(domain, tag)
                if tagged_fdt:
                    data_type = tagged_fdt
                # support old usage with 'UID'
                elif 'UID' in dt and dt['UID']:
                    data_type = FixtureDataType.get(dt['UID'])
                else:
                    data_type = new_data_type
                    pass
                if replace:
                    data_type.recursive_delete(transaction)
                    data_type = new_data_type
                data_type.fields = type_fields_with_properties
                data_type.is_global = dt.get('is_global', False)
                assert data_type.doc_type == FixtureDataType._doc_type
                if data_type.domain != domain:
                    data_type = new_data_type
                    messages.error(request, _("'%(UID)s' is not a valid UID. But the new type is created.") % {'UID': dt['UID']})
                if dt[DELETE_HEADER] == "Y" or dt[DELETE_HEADER] == "y":
                    data_type.recursive_delete(transaction)
                    continue
            except (ResourceNotFound, KeyError) as e:
                data_type = new_data_type
            transaction.save(data_type)

            data_items = workbook.get_worksheet(data_type.tag)
            for sort_key, di in enumerate(data_items):
                # Check that type definitions in 'types' sheet vs corresponding columns in the item-sheet MATCH
                item_fields_list = di['field'].keys()
                not_in_sheet, not_in_types = diff_lists(item_fields_list, data_type.fields_without_attributes)
                if len(not_in_sheet) > 0:
                    error_message = failure_messages["has_no_field_column"].format(tag=tag, field=not_in_sheet[0])
                    raise ExcelMalformatException(_(error_message))
                if len(not_in_types) > 0:
                    error_message = failure_messages["has_extra_column"].format(tag=tag, field=not_in_types[0])
                    raise ExcelMalformatException(_(error_message))

                # check that properties in 'types' sheet vs item-sheet MATCH
                for field in data_type.fields:
                    if len(field.properties) > 0:
                        sheet_props = di.get(field.field_name, {})
                        sheet_props_list = sheet_props.keys()
                        type_props = field.properties
                        not_in_sheet, not_in_types = diff_lists(sheet_props_list, type_props)
                        if len(not_in_sheet) > 0:
                            error_message = failure_messages["sheet_has_no_property"].format(
                                tag=tag,
                                property=not_in_sheet[0],
                                field=field.field_name
                            )
                            raise ExcelMalformatException(_(error_message))
                        if len(not_in_types) > 0:
                            error_message = failure_messages["sheet_has_extra_property"].format(
                                tag=tag,
                                property=not_in_types[0],
                                field=field.field_name
                            )
                            raise ExcelMalformatException(_(error_message))
                        # check that fields with properties are numbered
                        if type(di['field'][field.field_name]) != list:
                            error_message = failure_messages["invalid_field_with_property"].format(field=field.field_name)
                            raise ExcelMalformatException(_(error_message))
                        field_prop_len = len(di['field'][field.field_name])
                        for prop in sheet_props:
                            if type(sheet_props[prop]) != list:
                                error_message = failure_messages["invalid_property"].format(
                                    field=field.field_name,
                                    prop=prop
                                )
                                raise ExcelMalformatException(_(error_message))
                            if len(sheet_props[prop]) != field_prop_len:
                                error_message = failure_messages["wrong_field_property_combos"].format(
                                    field=field.field_name,
                                    prop=prop
                                )
                                raise ExcelMalformatException(_(error_message))

                # excel format check should have been covered by this line. Can make assumptions about data now
                type_fields = data_type.fields
                item_fields = {}
                for field in type_fields:
                    # if field doesn't have properties
                    if len(field.properties) == 0:
                        item_fields[field.field_name] = FieldList(
                            field_list=[FixtureItemField(
                                # using unicode here, to cast ints, and multi-language strings
                                field_value=unicode(di['field'][field.field_name]),
                                properties={}
                            )]
                        )
                    else:
                        field_list = []
                        field_prop_combos = di['field'][field.field_name]
                        prop_combo_len = len(field_prop_combos)
                        prop_dict = di[field.field_name]
                        for x in range(0, prop_combo_len):
                            fix_item_field = FixtureItemField(
                                field_value=unicode(field_prop_combos[x]),
                                properties={prop: unicode(prop_dict[prop][x]) for prop in prop_dict}
                            )
                            field_list.append(fix_item_field)
                        item_fields[field.field_name] = FieldList(
                            field_list=field_list
                        )

                new_data_item = FixtureDataItem(
                    domain=domain,
                    data_type_id=data_type.get_id,
                    fields=item_fields,
                    sort_key=sort_key
                )
                try:
                    if di['UID'] and not replace:
                        old_data_item = FixtureDataItem.get(di['UID'])
                    else:
                        old_data_item = new_data_item
                        pass
                    old_data_item.fields = item_fields   
                    if old_data_item.domain != domain or not old_data_item.data_type_id == data_type.get_id:
                        old_data_item = new_data_item
                        messages.error(request, _("'%(UID)s' is not a valid UID. But the new item is created.") % {'UID': di['UID']})
                    assert old_data_item.doc_type == FixtureDataItem._doc_type
                    if di[DELETE_HEADER] == "Y" or di[DELETE_HEADER] == "y":
                        old_data_item.recursive_delete(transaction)
                        continue               
                except (ResourceNotFound, KeyError):
                    old_data_item = new_data_item
                transaction.save(old_data_item)

                old_groups = old_data_item.groups
                for group in old_groups:
                    old_data_item.remove_group(group)
                old_users = old_data_item.users
                for user in old_users:
                    old_data_item.remove_user(user)

                for group_name in di.get('group', []):
                    group = group_memoizer.by_name(group_name)
                    if group:
                        old_data_item.add_group(group, transaction=transaction)
                    else:
                        messages.error(request, _("Unknown group: '%(name)s'. But the row is successfully added") % {'name': group_name})

                for raw_username in di.get('user', []):
                    try:
                        username = normalize_username(raw_username, domain)
                    except ValidationError:
                        messages.error(request, _("Invalid username: '******'. Row is not added") % {'name': raw_username})
                        continue
                    user = CommCareUser.get_by_username(username)
                    if user:
                        old_data_item.add_user(user)
                    else:
                        messages.error(request, _("Unknown user: '******'. But the row is successfully added") % {'name': raw_username})

    return_val["number_of_fixtures"] = number_of_fixtures + 1
    return return_val
示例#40
0
def create_or_update_users_and_groups(domain, user_specs, group_specs):
    group_memoizer = GroupMemoizer(domain)
    ret = {"errors": [], "rows": []}
    for row in group_specs:
        group_name, case_sharing, reporting = row['name'], row['case-sharing'], row['reporting']
        try:
            group = group_memoizer.get_or_create_group(group_name)
        except MultipleResultsFound:
            ret["errors"].append("Multiple groups named: %s" % group_name)
        else:
            group.case_sharing = case_sharing
            group.reporting = reporting
    usernames = set()
    user_ids = set()

    try:
        for row in user_specs:
            data, group_names, name, password, phone_number, user_id, username = (
                row.get(k) for k in sorted(allowed_headers)
            )
            if isinstance(password, float):
                # almost certainly what was intended
                password = unicode(int(password))
            group_names = group_names or []
            try:
                username = normalize_username(username, domain)
            except TypeError:
                username = None
            status_row = {'username': raw_username(username) if username else None}
            status_row['row'] = row
            if username in usernames or user_id in user_ids:
                status_row['flag'] = 'repeat'
            elif not username and not user_id:
                status_row['flag'] = 'missing-data'
            else:
                try:
                    if username:
                        usernames.add(username)
                    if user_id:
                        user_ids.add(user_id)
                    if user_id:
                        user = CommCareUser.get_by_user_id(user_id, domain)
                    else:
                        user = CommCareUser.get_by_username(username)
                    if user:
                        if user.domain != domain:
                            raise Exception('User with username %r is somehow in domain %r' % (user.username, user.domain))
                        if username and user.username != username:
                            user.change_username(username)
                        if password:
                            user.set_password(password)
                        status_row['flag'] = 'updated'
                    else:
                        if not password:
                            raise Exception("Cannot create a new user with a blank password")
                        user = CommCareUser.create(domain, username, password, uuid=user_id or '')
                        status_row['flag'] = 'created'
                    if phone_number:
                        user.add_phone_number(_fmt_phone(phone_number), default=True)
                    if name:
                        user.set_full_name(name)
                    if data:
                        user.user_data.update(data)
                    user.save()
                    if password:
                        # Without this line, digest auth doesn't work.
                        # With this line, digest auth works.
                        # Other than that, I'm not sure what's going on
                        user.get_django_user().check_password(password)
                    for group in Group.by_user(user):
                        if group.name not in group_names:
                            group = group_memoizer.get_or_create_group(group.name)
                            group.remove_user(user)

                    for group_name in group_names:
                        try:
                            group_memoizer.get_group(group_name).add_user(user)
                        except Exception:
                            raise Exception("Can't add to group '%s' (try adding it to your spreadsheet)" % group_name)
                except Exception, e:
                    if isinstance(e, DatabaseError):
                        transaction.rollback()
                    status_row['flag'] = 'error: %s' % e
                    
            ret["rows"].append(status_row)
    finally:
        group_memoizer.save_all()
    
    return ret
    
示例#41
0
 def get_username(reg):
     username = re.search(r'<username>(.*)</username>', reg).group(1)
     return normalize_username(username)
示例#42
0
def create_or_update_users_and_groups(upload_domain, user_specs, upload_user, group_memoizer=None, update_progress=None):
    domain_info_by_domain = {}

    def _get_domain_info(domain):
        domain_info = domain_info_by_domain.get(domain)
        if domain_info:
            return domain_info
        if domain == upload_domain:
            domain_group_memoizer = group_memoizer or GroupMemoizer(domain)
        else:
            domain_group_memoizer = GroupMemoizer(domain)
        domain_group_memoizer.load_all()
        can_assign_locations = domain_has_privilege(domain, privileges.LOCATIONS)
        location_cache = None
        if can_assign_locations:
            location_cache = SiteCodeToLocationCache(domain)

        domain_obj = Domain.get_by_name(domain)
        allowed_group_names = [group.name for group in domain_group_memoizer.groups]
        roles_by_name = {role.name: role for role in UserRole.by_domain(domain)}
        domain_user_specs = [spec for spec in user_specs if spec.get('domain', upload_domain) == domain]
        validators = get_user_import_validators(
            domain_obj,
            domain_user_specs,
            allowed_group_names,
            list(roles_by_name),
            upload_domain
        )

        domain_info = DomainInfo(
            validators,
            can_assign_locations,
            location_cache,
            roles_by_name,
            domain_group_memoizer
        )
        domain_info_by_domain[domain] = domain_info
        return domain_info

    ret = {"errors": [], "rows": []}

    current = 0

    try:
        for row in user_specs:
            if update_progress:
                update_progress(current)
                current += 1

            username = row.get('username')
            domain = row.get('domain') or upload_domain
            username = normalize_username(str(username), domain) if username else None
            status_row = {
                'username': username,
                'row': row,
            }

            domain_info = _get_domain_info(domain)

            try:
                for validator in domain_info.validators:
                    validator(row)
            except UserUploadError as e:
                status_row['flag'] = str(e)
                ret['rows'].append(status_row)
                continue

            data = row.get('data')
            email = row.get('email')
            group_names = list(map(str, row.get('group') or []))
            language = row.get('language')
            name = row.get('name')
            password = row.get('password')
            phone_number = row.get('phone-number')
            uncategorized_data = row.get('uncategorized_data')
            user_id = row.get('user_id')
            location_codes = row.get('location_code') or []
            if location_codes and not isinstance(location_codes, list):
                location_codes = [location_codes]
            # ignore empty
            location_codes = [code for code in location_codes if code]
            role = row.get('role', None)
            web_user = row.get('web_user')

            try:
                password = str(password) if password else None

                is_active = spec_value_to_boolean_or_none(row, 'is_active')
                is_account_confirmed = spec_value_to_boolean_or_none(row, 'is_account_confirmed')
                send_account_confirmation_email = spec_value_to_boolean_or_none(row, 'send_confirmation_email')
                remove_web_user = spec_value_to_boolean_or_none(row, 'remove_web_user')

                if user_id:
                    user = CommCareUser.get_by_user_id(user_id, domain)
                    if not user:
                        raise UserUploadError(_(
                            "User with ID '{user_id}' not found"
                        ).format(user_id=user_id, domain=domain))

                    if username and user.username != username:
                        raise UserUploadError(_(
                            'Changing usernames is not supported: %(username)r to %(new_username)r'
                        ) % {'username': user.username, 'new_username': username})

                    # note: explicitly not including "None" here because that's the default value if not set.
                    # False means it was set explicitly to that value
                    if is_account_confirmed is False and not web_user:
                        raise UserUploadError(_(
                            "You can only set 'Is Account Confirmed' to 'False' on a new User."
                        ))

                    if is_password(password):
                        user.set_password(password)
                        # overwrite password in results so we do not save it to the db
                        status_row['row']['password'] = '******'
                    status_row['flag'] = 'updated'
                else:
                    kwargs = {}
                    if is_account_confirmed is not None and not web_user:
                        kwargs['is_account_confirmed'] = is_account_confirmed
                    user = CommCareUser.create(domain, username, password, created_by=upload_user,
                                               created_via=USER_CHANGE_VIA_BULK_IMPORTER, commit=False, **kwargs)
                    status_row['flag'] = 'created'

                if phone_number:
                    user.add_phone_number(_fmt_phone(phone_number), default=True)
                if name:
                    user.set_full_name(str(name))
                if data:
                    user.user_data.update(data)
                if uncategorized_data:
                    user.user_data.update(uncategorized_data)
                if language:
                    user.language = language
                if email:
                    user.email = email.lower()
                if is_active is not None:
                    user.is_active = is_active

                if domain_info.can_assign_locations:
                    # Do this here so that we validate the location code before we
                    # save any other information to the user, this way either all of
                    # the user's information is updated, or none of it
                    location_ids = []
                    for code in location_codes:
                        loc = get_location_from_site_code(code, domain_info.location_cache)
                        location_ids.append(loc.location_id)

                    locations_updated = set(user.assigned_location_ids) != set(location_ids)
                    primary_location_removed = (user.location_id and not location_ids or
                                                user.location_id not in location_ids)

                    if primary_location_removed:
                        user.unset_location(commit=False)
                    if locations_updated:
                        user.reset_locations(location_ids, commit=False)

                if role:
                    role_qualified_id = domain_info.roles_by_name[role].get_qualified_id()
                    user.set_role(domain, role_qualified_id)

                if web_user:
                    user.user_data.update({'login_as_user': web_user})

                user.save()

                if web_user:
                    if not upload_user.can_edit_web_users():
                        raise UserUploadError(_(
                            "Only users with the edit web users permission can upload web users"
                        ))
                    current_user = CouchUser.get_by_username(web_user)
                    if remove_web_user:
                        if not current_user or not current_user.is_member_of(domain):
                            raise UserUploadError(_(
                                "You cannot remove a web user that is not a member of this project. {web_user} is not a member.").format(web_user=web_user)
                            )
                        else:
                            current_user.delete_domain_membership(domain)
                            current_user.save()
                    else:
                        if not role:
                            raise UserUploadError(_(
                                "You cannot upload a web user without a role. {web_user} does not have a role").format(web_user=web_user)
                            )
                        if not current_user and is_account_confirmed:
                            raise UserUploadError(_(
                                "You can only set 'Is Account Confirmed' to 'True' on an existing Web User. {web_user} is a new username.").format(web_user=web_user)
                            )
                        if current_user and not current_user.is_member_of(domain) and is_account_confirmed:
                            current_user.add_as_web_user(domain, role=role_qualified_id, location_id=user.location_id)
                        elif not current_user or not current_user.is_member_of(domain):
                            invite_data = {
                                'email': web_user,
                                'invited_by': upload_user.user_id,
                                'invited_on': datetime.utcnow(),
                                'domain': domain,
                                'role': role_qualified_id,
                                'supply_point': user.location_id
                            }
                            invite = Invitation(**invite_data)
                            invite.save()
                            if send_account_confirmation_email:
                                invite.send_activation_email()

                        elif current_user.is_member_of(domain):
                            # edit existing user in the domain
                            current_user.set_role(domain, role_qualified_id)
                            if user.location_id:
                                current_user.set_location(domain, user.location_id)
                            else:
                                current_user.unset_location(domain)
                            current_user.save()

                if send_account_confirmation_email and not web_user:
                    send_account_confirmation_if_necessary(user)

                if is_password(password):
                    # Without this line, digest auth doesn't work.
                    # With this line, digest auth works.
                    # Other than that, I'm not sure what's going on
                    # Passing use_primary_db=True because of https://dimagi-dev.atlassian.net/browse/ICDS-465
                    user.get_django_user(use_primary_db=True).check_password(password)

                for group in domain_info.group_memoizer.by_user_id(user.user_id):
                    if group.name not in group_names:
                        group.remove_user(user)

                for group_name in group_names:
                    domain_info.group_memoizer.by_name(group_name).add_user(user, save=False)

            except (UserUploadError, CouchUser.Inconsistent) as e:
                status_row['flag'] = str(e)

            ret["rows"].append(status_row)
    finally:
        try:
            for domain_info in domain_info_by_domain.values():
                domain_info.group_memoizer.save_all()
        except BulkSaveError as e:
            _error_message = (
                "Oops! We were not able to save some of your group changes. "
                "Please make sure no one else is editing your groups "
                "and try again."
            )
            logging.exception((
                'BulkSaveError saving groups. '
                'User saw error message "%s". Errors: %s'
            ) % (_error_message, e.errors))
            ret['errors'].append(_error_message)

    return ret
示例#43
0
def run_upload(request, domain, workbook):
    return_val = {
        "unknown_groups": [], 
        "unknown_users": [], 
        "number_of_fixtures": 0,
    }
    group_memoizer = GroupMemoizer(domain)

    data_types = workbook.get_worksheet(title='types')

    def _get_or_raise(container, attr):
        try:
            return container[attr]
        except KeyError:
            raise Exception("Workbook 'types' has no column '{attr}'".format(attr=attr))
   
    number_of_fixtures = -1
    with CouchTransaction() as transaction:
        for number_of_fixtures, dt in enumerate(data_types):
            tag = _get_or_raise(dt, 'tag')
            type_definition_fields = _get_or_raise(dt, 'field')

            new_data_type = FixtureDataType(
                    domain=domain,
                    is_global=dt.get('is_global', False),
                    name=_get_or_raise(dt, 'name'),
                    tag=_get_or_raise(dt, 'tag'),
                    fields=type_definition_fields,
            )
            try:
                if dt['UID']:
                    data_type = FixtureDataType.get(dt['UID'])
                else:
                    data_type = new_data_type
                    pass
                data_type.fields = type_definition_fields
                data_type.is_global = dt.get('is_global', False)
                assert data_type.doc_type == FixtureDataType._doc_type
                if data_type.domain != domain:
                    data_type = new_data_type
                    messages.error(request, _("'%(UID)s' is not a valid UID. But the new type is created.") % {'UID': dt['UID']})
                if dt[DELETE_HEADER] == "Y" or dt[DELETE_HEADER] == "y":
                    data_type.recursive_delete(transaction)
                    continue
            except (ResourceNotFound, KeyError) as e:
                data_type = new_data_type
            transaction.save(data_type)

            data_items = workbook.get_worksheet(data_type.tag)
            for sort_key, di in enumerate(data_items):
                # Check that type definitions in 'types' sheet vs corresponding columns in the item-sheet MATCH
                item_fields = di['field']
                for field in type_definition_fields:
                    if not item_fields.has_key(field):
                        raise Exception(_("Workbook '%(tag)s' does not contain the column " +
                                          "'%(field)s' specified in its 'types' definition") % {'tag': tag, 'field': field})
                item_fields_list = di['field'].keys()
                for field in item_fields_list:
                    if not field in type_definition_fields:
                        raise Exception(_("""Workbook '%(tag)s' has an extra column 
                                          '%(field)s' that's not defined in its 'types' definition""") % {'tag': tag, 'field': field})                
                new_data_item = FixtureDataItem(
                    domain=domain,
                    data_type_id=data_type.get_id,
                    fields=item_fields,
                    sort_key=sort_key
                )
                try:
                    if di['UID']:
                        old_data_item = FixtureDataItem.get(di['UID'])
                    else:
                        old_data_item = new_data_item
                        pass
                    old_data_item.fields = di['field']   
                    if old_data_item.domain != domain:
                        old_data_item = new_data_item
                        messages.error(request, _("'%(UID)s' is not a valid UID. But the new item is created.") % {'UID': di['UID'] })
                    assert old_data_item.doc_type == FixtureDataItem._doc_type
                    assert old_data_item.data_type_id == data_type.get_id
                    if di[DELETE_HEADER] == "Y" or di[DELETE_HEADER] == "y":
                        old_data_item.recursive_delete(transaction)
                        continue               
                except (ResourceNotFound, KeyError) as e:
                    old_data_item = new_data_item
                transaction.save(old_data_item)

                old_groups = old_data_item.get_groups()
                for group in old_groups:
                    old_data_item.remove_group(group)
                old_users = old_data_item.get_users()
                for user in old_users:
                    old_data_item.remove_user(user)

                for group_name in di.get('group', []):
                        group = group_memoizer.by_name(group_name)
                        if group:
                            old_data_item.add_group(group, transaction=transaction)
                        else:
                            messages.error(request, _("Unknown group: '%(name)s'. But the row is successfully added") % {'name': group_name})

                for raw_username in di.get('user', []):
                        try:
                            username = normalize_username(raw_username, domain)
                        except ValidationError:
                            messages.error(request, _("Invalid username: '******'. Row is not added") % {'name': raw_username})
                            continue
                        user = CommCareUser.get_by_username(username)
                        if user:
                            old_data_item.add_user(user)
                        else:
                            messages.error(request, _("Unknown user: '******'. But the row is successfully added") % {'name': raw_username})

    return_val["number_of_fixtures"] = number_of_fixtures + 1
    return return_val
示例#44
0
    def setUpClass(cls):
        super(SchedulingRecipientTest, cls).setUpClass()

        cls.domain_obj = create_domain(cls.domain)

        cls.location_types = setup_location_types(cls.domain,
                                                  ['country', 'state', 'city'])
        cls.country_location = make_loc('usa',
                                        domain=cls.domain,
                                        type='country')
        cls.state_location = make_loc('ma',
                                      domain=cls.domain,
                                      type='state',
                                      parent=cls.country_location)
        cls.city_location = make_loc('boston',
                                     domain=cls.domain,
                                     type='city',
                                     parent=cls.state_location)

        cls.mobile_user = CommCareUser.create(cls.domain, 'mobile', 'abc',
                                              None, None)
        cls.mobile_user.set_location(cls.city_location)

        cls.mobile_user2 = CommCareUser.create(cls.domain, 'mobile2', 'abc',
                                               None, None)
        cls.mobile_user2.set_location(cls.state_location)

        cls.mobile_user3 = CommCareUser.create(cls.domain,
                                               'mobile3',
                                               'abc',
                                               None,
                                               None,
                                               metadata={
                                                   'role': 'pharmacist',
                                               })
        cls.mobile_user3.save()

        cls.mobile_user4 = CommCareUser.create(cls.domain,
                                               'mobile4',
                                               'abc',
                                               None,
                                               None,
                                               metadata={
                                                   'role': 'nurse',
                                               })
        cls.mobile_user4.save()

        cls.mobile_user5 = CommCareUser.create(cls.domain,
                                               'mobile5',
                                               'abc',
                                               None,
                                               None,
                                               metadata={
                                                   'role':
                                                   ['nurse', 'pharmacist'],
                                               })
        cls.mobile_user5.save()

        full_username = normalize_username('mobile', cls.domain)
        cls.full_mobile_user = CommCareUser.create(cls.domain, full_username,
                                                   'abc', None, None)

        cls.definition = CustomDataFieldsDefinition(
            domain=cls.domain, field_type=UserFieldsView.field_type)
        cls.definition.save()
        cls.definition.set_fields([
            Field(
                slug='role',
                label='Role',
            ),
        ])
        cls.definition.save()
        cls.profile = CustomDataFieldsProfile(
            name='nurse_profile',
            fields={'role': ['nurse']},
            definition=cls.definition,
        )
        cls.profile.save()
        cls.mobile_user6 = CommCareUser.create(cls.domain,
                                               'mobile6',
                                               'abc',
                                               None,
                                               None,
                                               metadata={
                                                   PROFILE_SLUG:
                                                   cls.profile.id,
                                               })
        cls.mobile_user5.save()

        cls.web_user = WebUser.create(cls.domain, 'web', 'abc', None, None)

        cls.web_user2 = WebUser.create(cls.domain,
                                       'web2',
                                       'abc',
                                       None,
                                       None,
                                       metadata={
                                           'role': 'nurse',
                                       })
        cls.web_user2.save()

        cls.group = Group(domain=cls.domain, users=[cls.mobile_user.get_id])
        cls.group.save()

        cls.group2 = Group(domain=cls.domain,
                           users=[
                               cls.mobile_user.get_id,
                               cls.mobile_user3.get_id,
                               cls.mobile_user4.get_id,
                               cls.mobile_user5.get_id,
                               cls.mobile_user6.get_id,
                           ])
        cls.group2.save()

        cls.case_group = CommCareCaseGroup(domain=cls.domain)
        cls.case_group.save()

        cls.process_pillow_changes = process_pillow_changes(
            'DefaultChangeFeedPillow')
        cls.process_pillow_changes.add_pillow(get_case_messaging_sync_pillow())
示例#45
0
def run_upload(domain, workbook, replace=False, task=None):
    return_val = FixtureUploadResult()
    group_memoizer = GroupMemoizer(domain)

    pre_populate_location_groups(group_memoizer, domain)

    def diff_lists(list_a, list_b):
        set_a = set(list_a)
        set_b = set(list_b)
        not_in_b = set_a.difference(set_b)
        not_in_a = set_a.difference(set_a)
        return list(not_in_a), list(not_in_b)

    with CouchTransaction() as transaction:
        type_sheets = workbook.get_all_type_sheets()
        total_tables = len(type_sheets)
        return_val.number_of_fixtures = total_tables

        def _update_progress(table_count, item_count, items_in_table):
            if task:
                processed = table_count * 10 + (10. * item_count / items_in_table)
                DownloadBase.set_progress(task, processed, 10 * total_tables)

        for table_number, table_def in enumerate(type_sheets):
            tag = table_def.table_id
            new_data_type = FixtureDataType(
                domain=domain,
                is_global=table_def.is_global,
                tag=table_def.table_id,
                fields=table_def.fields,
                item_attributes=table_def.item_attributes
            )
            try:
                tagged_fdt = FixtureDataType.fixture_tag_exists(domain, tag)
                if tagged_fdt:
                    data_type = tagged_fdt
                # support old usage with 'UID'
                elif table_def.uid:
                    data_type = FixtureDataType.get(table_def.uid)
                else:
                    data_type = new_data_type

                if replace and data_type != new_data_type:
                    data_type.recursive_delete(transaction)
                    data_type = new_data_type

                data_type.fields = table_def.fields
                data_type.item_attributes = table_def.item_attributes
                data_type.is_global = table_def.is_global
                assert data_type.doc_type == FixtureDataType._doc_type
                if data_type.domain != domain:
                    data_type = new_data_type
                    return_val.errors.append(_("'%(UID)s' is not a valid UID. But the new type is created.") % {'UID': table_def.uid})
                if table_def.delete:
                    data_type.recursive_delete(transaction)
                    continue
            except (ResourceNotFound, KeyError) as e:
                data_type = new_data_type
            transaction.save(data_type)

            data_items = list(workbook.get_data_sheet(data_type))
            items_in_table = len(data_items)
            for sort_key, di in enumerate(data_items):
                _update_progress(table_number, sort_key, items_in_table)
                # Check that type definitions in 'types' sheet vs corresponding columns in the item-sheet MATCH
                item_fields_list = di['field'].keys() if 'field' in di else []
                not_in_sheet, not_in_types = diff_lists(item_fields_list, data_type.fields_without_attributes)
                if len(not_in_sheet) > 0:
                    error_message = _(FAILURE_MESSAGES["has_no_field_column"]).format(tag=tag, field=not_in_sheet[0])
                    raise ExcelMalformatException(error_message)
                if len(not_in_types) > 0:
                    error_message = _(FAILURE_MESSAGES["has_extra_column"]).format(tag=tag, field=not_in_types[0])
                    raise ExcelMalformatException(error_message)

                # check that this item has all the properties listed in its 'types' definition
                item_attributes_list = di['property'].keys() if 'property' in di else []
                not_in_sheet, not_in_types = diff_lists(item_attributes_list, data_type.item_attributes)
                if len(not_in_sheet) > 0:
                    error_message = _(FAILURE_MESSAGES["has_no_field_column"]).format(tag=tag, field=not_in_sheet[0])
                    raise ExcelMalformatException(error_message)
                if len(not_in_types) > 0:
                    error_message = _(FAILURE_MESSAGES["has_extra_column"]).format(tag=tag, field=not_in_types[0])
                    raise ExcelMalformatException(error_message)

                # check that properties in 'types' sheet vs item-sheet MATCH
                for field in data_type.fields:
                    if len(field.properties) > 0:
                        sheet_props = di.get(field.field_name, {})
                        sheet_props_list = sheet_props.keys()
                        type_props = field.properties
                        not_in_sheet, not_in_types = diff_lists(sheet_props_list, type_props)
                        if len(not_in_sheet) > 0:
                            error_message = _(FAILURE_MESSAGES["sheet_has_no_property"]).format(
                                tag=tag,
                                property=not_in_sheet[0],
                                field=field.field_name
                            )
                            raise ExcelMalformatException(error_message)
                        if len(not_in_types) > 0:
                            error_message = _(FAILURE_MESSAGES["sheet_has_extra_property"]).format(
                                tag=tag,
                                property=not_in_types[0],
                                field=field.field_name
                            )
                            raise ExcelMalformatException(error_message)
                        # check that fields with properties are numbered
                        if type(di['field'][field.field_name]) != list:
                            error_message = _(FAILURE_MESSAGES["invalid_field_with_property"]).format(field=field.field_name)
                            raise ExcelMalformatException(error_message)
                        field_prop_len = len(di['field'][field.field_name])
                        for prop in sheet_props:
                            if type(sheet_props[prop]) != list:
                                error_message = _(FAILURE_MESSAGES["invalid_property"]).format(
                                    field=field.field_name,
                                    prop=prop
                                )
                                raise ExcelMalformatException(error_message)
                            if len(sheet_props[prop]) != field_prop_len:
                                error_message = _(FAILURE_MESSAGES["wrong_field_property_combos"]).format(
                                    field=field.field_name,
                                    prop=prop
                                )
                                raise ExcelMalformatException(error_message)

                # excel format check should have been covered by this line. Can make assumptions about data now
                type_fields = data_type.fields
                item_fields = {}
                for field in type_fields:
                    # if field doesn't have properties
                    if len(field.properties) == 0:
                        item_fields[field.field_name] = FieldList(
                            field_list=[FixtureItemField(
                                # using unicode here, to cast ints, and multi-language strings
                                field_value=unicode(di['field'][field.field_name]),
                                properties={}
                            )]
                        )
                    else:
                        field_list = []
                        field_prop_combos = di['field'][field.field_name]
                        prop_combo_len = len(field_prop_combos)
                        prop_dict = di[field.field_name]
                        for x in range(0, prop_combo_len):
                            fix_item_field = FixtureItemField(
                                field_value=unicode(field_prop_combos[x]),
                                properties={prop: unicode(prop_dict[prop][x]) for prop in prop_dict}
                            )
                            field_list.append(fix_item_field)
                        item_fields[field.field_name] = FieldList(
                            field_list=field_list
                        )

                item_attributes = di.get('property', {})
                new_data_item = FixtureDataItem(
                    domain=domain,
                    data_type_id=data_type.get_id,
                    fields=item_fields,
                    item_attributes=item_attributes,
                    sort_key=sort_key
                )
                try:
                    if di['UID'] and not replace:
                        old_data_item = FixtureDataItem.get(di['UID'])
                    else:
                        old_data_item = new_data_item
                        pass
                    old_data_item.fields = item_fields
                    old_data_item.item_attributes = item_attributes
                    if old_data_item.domain != domain or not old_data_item.data_type_id == data_type.get_id:
                        old_data_item = new_data_item
                        return_val.errors.append(_("'%(UID)s' is not a valid UID. But the new item is created.") % {'UID': di['UID']})
                    assert old_data_item.doc_type == FixtureDataItem._doc_type
                    if di[DELETE_HEADER] == "Y" or di[DELETE_HEADER] == "y":
                        old_data_item.recursive_delete(transaction)
                        continue
                except (ResourceNotFound, KeyError):
                    old_data_item = new_data_item
                transaction.save(old_data_item)

                old_groups = old_data_item.groups
                for group in old_groups:
                    old_data_item.remove_group(group)
                old_users = old_data_item.users
                for user in old_users:
                    old_data_item.remove_user(user)

                for group_name in di.get('group', []):
                    group = group_memoizer.by_name(group_name)
                    if group:
                        old_data_item.add_group(group, transaction=transaction)
                    else:
                        return_val.errors.append(_("Unknown group: '%(name)s'. But the row is successfully added") % {'name': group_name})

                for raw_username in di.get('user', []):
                    try:
                        username = normalize_username(str(raw_username), domain)
                    except ValidationError:
                        return_val.errors.append(_("Invalid username: '******'. Row is not added") % {'name': raw_username})
                        continue
                    user = CommCareUser.get_by_username(username)
                    if user:
                        old_data_item.add_user(user)
                    else:
                        return_val.errors.append(_("Unknown user: '******'. But the row is successfully added") % {'name': raw_username})

    return return_val
示例#46
0
def create_or_update_users_and_groups(domain, user_specs, group_specs):
    ret = {"errors": [], "rows": []}
    group_memoizer = create_or_update_groups(domain, group_specs, log=ret)
    usernames = set()
    user_ids = set()

    allowed_groups = set(group_memoizer.groups)
    allowed_group_names = [group.name for group in allowed_groups]

    try:
        for row in user_specs:
            data, group_names, language, name, password, phone_number, user_id, username = (
                row.get(k) for k in sorted(allowed_headers)
            )
            password = unicode(password)
            group_names = group_names or []
            try:
                username = normalize_username(username, domain)
            except TypeError:
                username = None
            except ValidationError:
                ret['rows'].append({
                    'username': username,
                    'row': row,
                    'flag': _('username cannot contain spaces or symbols'),
                })
                continue
            status_row = {
                'username': raw_username(username) if username else None,
                'row': row,
            }
            if username in usernames or user_id in user_ids:
                status_row['flag'] = 'repeat'
            elif not username and not user_id:
                status_row['flag'] = 'missing-data'
            else:
                try:
                    if username:
                        usernames.add(username)
                    if user_id:
                        user_ids.add(user_id)
                    if user_id:
                        user = CommCareUser.get_by_user_id(user_id, domain)
                    else:
                        user = CommCareUser.get_by_username(username)
                    if user:
                        if user.domain != domain:
                            raise UserUploadError(_('User with username %(username)r is somehow in domain %(domain)r') %
                                            {'username': user.username, 'domain': user.domain})
                        if username and user.username != username:
                            user.change_username(username)
                        if password:
                            user.set_password(password)
                        status_row['flag'] = 'updated'
                    else:
                        if not password:
                            raise UserUploadError(_("Cannot create a new user with a blank password"))
                        user = CommCareUser.create(domain, username, password, uuid=user_id or '')
                        status_row['flag'] = 'created'
                    if phone_number:
                        user.add_phone_number(_fmt_phone(phone_number), default=True)
                    if name:
                        user.set_full_name(name)
                    if data:
                        user.user_data.update(data)
                    if language:
                        user.language = language
                    user.save()
                    if password:
                        # Without this line, digest auth doesn't work.
                        # With this line, digest auth works.
                        # Other than that, I'm not sure what's going on
                        user.get_django_user().check_password(password)

                    for group_id in Group.by_user(user, wrap=False):
                        group = group_memoizer.get(group_id)
                        if group.name not in group_names:
                            group.remove_user(user, save=False)

                    for group_name in group_names:
                        if group_name not in allowed_group_names:
                            raise UserUploadError(_(
                                "Can't add to group '%s' "
                                "(try adding it to your spreadsheet)"
                            ) % group_name)
                        group_memoizer.by_name(group_name).add_user(user, save=False)

                except UserUploadError as e:
                    status_row['flag'] = '%s' % e
                    
            ret["rows"].append(status_row)
    finally:
        group_memoizer.save_all()
    
    return ret
示例#47
0
 def get_username(reg):
     username = re.search(r'<username>(.*)</username>', reg).group(1)
     return normalize_username(username)
示例#48
0
 def test_username_doesnt_have_domain(self):
     user = create_restore_user(
         username=normalize_username('withdomain', domain='thedomain'))
     restore_payload = xml.get_registration_xml(user)
     self.assertTrue('thedomain' not in restore_payload)
示例#49
0
def create_or_update_users_and_groups(domain,
                                      user_specs,
                                      group_specs,
                                      location_specs,
                                      task=None):
    ret = {"errors": [], "rows": []}
    total = len(user_specs) + len(group_specs) + len(location_specs)

    def _set_progress(progress):
        if task is not None:
            DownloadBase.set_progress(task, progress, total)

    group_memoizer = create_or_update_groups(domain, group_specs, log=ret)
    current = len(group_specs)

    usernames = set()
    user_ids = set()
    allowed_groups = set(group_memoizer.groups)
    allowed_group_names = [group.name for group in allowed_groups]
    try:
        for row in user_specs:
            _set_progress(current)
            current += 1
            data, email, group_names, language, name, password, phone_number, user_id, username = (
                row.get(k) for k in sorted(allowed_headers))
            if password:
                password = unicode(password)
            group_names = group_names or []
            try:
                username = normalize_username(str(username), domain)
            except TypeError:
                username = None
            except ValidationError:
                ret['rows'].append({
                    'username':
                    username,
                    'row':
                    row,
                    'flag':
                    _('username cannot contain spaces or symbols'),
                })
                continue
            status_row = {
                'username': raw_username(username) if username else None,
                'row': row,
            }
            if username in usernames or user_id in user_ids:
                status_row['flag'] = 'repeat'
            elif not username and not user_id:
                status_row['flag'] = 'missing-data'
            else:
                try:
                    if username:
                        usernames.add(username)
                    if user_id:
                        user_ids.add(user_id)
                    if user_id:
                        user = CommCareUser.get_by_user_id(user_id, domain)
                    else:
                        user = CommCareUser.get_by_username(username)

                    def is_password(password):
                        if not password:
                            return False
                        for c in password:
                            if c != "*":
                                return True
                        return False

                    if user:
                        if user.domain != domain:
                            raise UserUploadError(
                                _('User with username %(username)r is '
                                  'somehow in domain %(domain)r') % {
                                      'username': user.username,
                                      'domain': user.domain
                                  })
                        if username and user.username != username:
                            user.change_username(username)
                        if is_password(password):
                            user.set_password(password)
                        status_row['flag'] = 'updated'
                    else:
                        if len(raw_username(username)
                               ) > CommCareAccountForm.max_len_username:
                            ret['rows'].append({
                                'username':
                                username,
                                'row':
                                row,
                                'flag':
                                _("username cannot contain greater than %d characters"
                                  % CommCareAccountForm.max_len_username)
                            })
                            continue
                        if not is_password(password):
                            raise UserUploadError(
                                _("Cannot create a new user with a blank password"
                                  ))
                        user = CommCareUser.create(domain,
                                                   username,
                                                   password,
                                                   uuid=user_id or '',
                                                   commit=False)
                        status_row['flag'] = 'created'
                    if phone_number:
                        user.add_phone_number(_fmt_phone(phone_number),
                                              default=True)
                    if name:
                        user.set_full_name(name)
                    if data:
                        user.user_data.update(data)
                    if language:
                        user.language = language
                    if email:
                        user.email = email
                    user.save()
                    if is_password(password):
                        # Without this line, digest auth doesn't work.
                        # With this line, digest auth works.
                        # Other than that, I'm not sure what's going on
                        user.get_django_user().check_password(password)

                    for group_id in Group.by_user(user, wrap=False):
                        group = group_memoizer.get(group_id)
                        if group.name not in group_names:
                            group.remove_user(user, save=False)

                    for group_name in group_names:
                        if group_name not in allowed_group_names:
                            raise UserUploadError(
                                _("Can't add to group '%s' "
                                  "(try adding it to your spreadsheet)") %
                                group_name)
                        group_memoizer.by_name(group_name).add_user(user,
                                                                    save=False)

                except (UserUploadError, CouchUser.Inconsistent) as e:
                    status_row['flag'] = unicode(e)

            ret["rows"].append(status_row)
    finally:
        try:
            group_memoizer.save_all()
        except BulkSaveError as e:
            _error_message = (
                "Oops! We were not able to save some of your group changes. "
                "Please make sure no one else is editing your groups "
                "and try again.")
            logging.exception(('BulkSaveError saving groups. '
                               'User saw error message "%s". Errors: %s') %
                              (_error_message, e.errors))
            ret['errors'].append(_error_message)

    create_or_update_locations(domain, location_specs, log=ret)
    _set_progress(total)
    return ret
示例#50
0
def run_upload(request, domain, workbook, replace=False):
    return_val = {
        "unknown_groups": [],
        "unknown_users": [],
        "number_of_fixtures": 0,
    }
    failure_messages = {
        "has_no_column":
        "Workbook 'types' has no column '{column_name}'.",
        "has_no_field_column":
        "Excel-sheet '{tag}' does not contain the column '{field}' "
        "as specified in its 'types' definition",
        "has_extra_column":
        "Excel-sheet '{tag}' has an extra column" +
        "'{field}' that's not defined in its 'types' definition",
        "wrong_property_syntax":
        "Properties should be specified as 'field 1: property 1'. In 'types' sheet, "
        + "'{prop_key}' for field '{field}' is not correctly formatted",
        "sheet_has_no_property":
        "Excel-sheet '{tag}' does not contain property " +
        "'{property}' of the field '{field}' as specified in its 'types' definition",
        "sheet_has_extra_property":
        "Excel-sheet '{tag}'' has an extra property " +
        "'{property}' for the field '{field}' that's not defined in its 'types' definition. Re-check the formatting",
        "invalid_field_with_property":
        "Fields with attributes should be numbered as 'field: {field} integer",
        "invalid_property":
        "Attribute should be written as '{field}: {prop} interger'",
        "wrong_field_property_combos":
        "Number of values for field '{field}' and attribute '{prop}' should be same",
        "replace_with_UID":
        "Rows shouldn't contain UIDs while using replace option. Excel sheet '{tag}' contains UID in a row."
    }

    group_memoizer = GroupMemoizer(domain)

    data_types = workbook.get_worksheet(title='types')

    def _get_or_raise(container, attr):
        try:
            return container[attr]
        except KeyError:
            raise ExcelMalformatException(
                _(failure_messages["has_no_column"].format(column_name=attr)))

    def diff_lists(list_a, list_b):
        set_a = set(list_a)
        set_b = set(list_b)
        not_in_b = set_a.difference(set_b)
        not_in_a = set_a.difference(set_a)
        return list(not_in_a), list(not_in_b)

    number_of_fixtures = -1
    with CouchTransaction() as transaction:
        fixtures_tags = []
        type_sheets = []
        for number_of_fixtures, dt in enumerate(data_types):
            try:
                tag = _get_or_raise(dt, 'table_id')
            except ExcelMalformatException:
                tag = _get_or_raise(dt, 'tag')
            if tag in fixtures_tags:
                error_message = "Upload Failed: Lookup-tables should have unique 'table_id'. There are two rows with table_id '{tag}' in 'types' sheet."
                raise DuplicateFixtureTagException(
                    _(error_message.format(tag=tag)))
            fixtures_tags.append(tag)
            type_sheets.append(dt)
        for number_of_fixtures, dt in enumerate(type_sheets):
            try:
                tag = _get_or_raise(dt, 'table_id')
            except ExcelMalformatException:
                messages.info(
                    request,
                    _("Excel-header 'tag' is renamed as 'table_id' and 'name' header is no longer needed."
                      ))
                tag = _get_or_raise(dt, 'tag')

            type_definition_fields = _get_or_raise(dt, 'field')
            type_fields_with_properties = []
            for count, field in enumerate(type_definition_fields):
                prop_key = "field " + str(count + 1)
                if dt.has_key(prop_key):
                    try:
                        property_list = dt[prop_key]["property"]
                    except KeyError:
                        error_message = failure_messages[
                            "wrong_property_syntax"].format(prop_key=prop_key,
                                                            field=field)
                        raise ExcelMalformatException(_(error_message))
                else:
                    property_list = []
                field_with_prop = FixtureTypeField(field_name=field,
                                                   properties=property_list)
                type_fields_with_properties.append(field_with_prop)

            new_data_type = FixtureDataType(
                domain=domain,
                is_global=dt.get('is_global', False),
                tag=tag,
                fields=type_fields_with_properties,
            )
            try:
                tagged_fdt = FixtureDataType.fixture_tag_exists(domain, tag)
                if tagged_fdt:
                    data_type = tagged_fdt
                # support old usage with 'UID'
                elif 'UID' in dt and dt['UID']:
                    data_type = FixtureDataType.get(dt['UID'])
                else:
                    data_type = new_data_type
                    pass
                if replace:
                    data_type.recursive_delete(transaction)
                    data_type = new_data_type
                data_type.fields = type_fields_with_properties
                data_type.is_global = dt.get('is_global', False)
                assert data_type.doc_type == FixtureDataType._doc_type
                if data_type.domain != domain:
                    data_type = new_data_type
                    messages.error(
                        request,
                        _("'%(UID)s' is not a valid UID. But the new type is created."
                          ) % {'UID': dt['UID']})
                if dt[DELETE_HEADER] == "Y" or dt[DELETE_HEADER] == "y":
                    data_type.recursive_delete(transaction)
                    continue
            except (ResourceNotFound, KeyError) as e:
                data_type = new_data_type
            transaction.save(data_type)

            data_items = workbook.get_worksheet(data_type.tag)
            for sort_key, di in enumerate(data_items):
                # Check that type definitions in 'types' sheet vs corresponding columns in the item-sheet MATCH
                item_fields_list = di['field'].keys()
                not_in_sheet, not_in_types = diff_lists(
                    item_fields_list, data_type.fields_without_attributes)
                if len(not_in_sheet) > 0:
                    error_message = failure_messages[
                        "has_no_field_column"].format(tag=tag,
                                                      field=not_in_sheet[0])
                    raise ExcelMalformatException(_(error_message))
                if len(not_in_types) > 0:
                    error_message = failure_messages[
                        "has_extra_column"].format(tag=tag,
                                                   field=not_in_types[0])
                    raise ExcelMalformatException(_(error_message))

                # check that properties in 'types' sheet vs item-sheet MATCH
                for field in data_type.fields:
                    if len(field.properties) > 0:
                        sheet_props = di.get(field.field_name, {})
                        sheet_props_list = sheet_props.keys()
                        type_props = field.properties
                        not_in_sheet, not_in_types = diff_lists(
                            sheet_props_list, type_props)
                        if len(not_in_sheet) > 0:
                            error_message = failure_messages[
                                "sheet_has_no_property"].format(
                                    tag=tag,
                                    property=not_in_sheet[0],
                                    field=field.field_name)
                            raise ExcelMalformatException(_(error_message))
                        if len(not_in_types) > 0:
                            error_message = failure_messages[
                                "sheet_has_extra_property"].format(
                                    tag=tag,
                                    property=not_in_types[0],
                                    field=field.field_name)
                            raise ExcelMalformatException(_(error_message))
                        # check that fields with properties are numbered
                        if type(di['field'][field.field_name]) != list:
                            error_message = failure_messages[
                                "invalid_field_with_property"].format(
                                    field=field.field_name)
                            raise ExcelMalformatException(_(error_message))
                        field_prop_len = len(di['field'][field.field_name])
                        for prop in sheet_props:
                            if type(sheet_props[prop]) != list:
                                error_message = failure_messages[
                                    "invalid_property"].format(
                                        field=field.field_name, prop=prop)
                                raise ExcelMalformatException(_(error_message))
                            if len(sheet_props[prop]) != field_prop_len:
                                error_message = failure_messages[
                                    "wrong_field_property_combos"].format(
                                        field=field.field_name, prop=prop)
                                raise ExcelMalformatException(_(error_message))

                # excel format check should have been covered by this line. Can make assumptions about data now
                type_fields = data_type.fields
                item_fields = {}
                for field in type_fields:
                    # if field doesn't have properties
                    if len(field.properties) == 0:
                        item_fields[field.field_name] = FieldList(field_list=[
                            FixtureItemField(
                                # using unicode here, to cast ints, and multi-language strings
                                field_value=unicode(di['field'][
                                    field.field_name]),
                                properties={})
                        ])
                    else:
                        field_list = []
                        field_prop_combos = di['field'][field.field_name]
                        prop_combo_len = len(field_prop_combos)
                        prop_dict = di[field.field_name]
                        for x in range(0, prop_combo_len):
                            fix_item_field = FixtureItemField(
                                field_value=unicode(field_prop_combos[x]),
                                properties={
                                    prop: unicode(prop_dict[prop][x])
                                    for prop in prop_dict
                                })
                            field_list.append(fix_item_field)
                        item_fields[field.field_name] = FieldList(
                            field_list=field_list)

                new_data_item = FixtureDataItem(domain=domain,
                                                data_type_id=data_type.get_id,
                                                fields=item_fields,
                                                sort_key=sort_key)
                try:
                    if di['UID'] and not replace:
                        old_data_item = FixtureDataItem.get(di['UID'])
                    else:
                        old_data_item = new_data_item
                        pass
                    old_data_item.fields = item_fields
                    if old_data_item.domain != domain or not old_data_item.data_type_id == data_type.get_id:
                        old_data_item = new_data_item
                        messages.error(
                            request,
                            _("'%(UID)s' is not a valid UID. But the new item is created."
                              ) % {'UID': di['UID']})
                    assert old_data_item.doc_type == FixtureDataItem._doc_type
                    if di[DELETE_HEADER] == "Y" or di[DELETE_HEADER] == "y":
                        old_data_item.recursive_delete(transaction)
                        continue
                except (ResourceNotFound, KeyError):
                    old_data_item = new_data_item
                transaction.save(old_data_item)

                old_groups = old_data_item.groups
                for group in old_groups:
                    old_data_item.remove_group(group)
                old_users = old_data_item.users
                for user in old_users:
                    old_data_item.remove_user(user)

                for group_name in di.get('group', []):
                    group = group_memoizer.by_name(group_name)
                    if group:
                        old_data_item.add_group(group, transaction=transaction)
                    else:
                        messages.error(
                            request,
                            _("Unknown group: '%(name)s'. But the row is successfully added"
                              ) % {'name': group_name})

                for raw_username in di.get('user', []):
                    try:
                        username = normalize_username(str(raw_username),
                                                      domain)
                    except ValidationError:
                        messages.error(
                            request,
                            _("Invalid username: '******'. Row is not added")
                            % {'name': raw_username})
                        continue
                    user = CommCareUser.get_by_username(username)
                    if user:
                        old_data_item.add_user(user)
                    else:
                        messages.error(
                            request,
                            _("Unknown user: '******'. But the row is successfully added"
                              ) % {'name': raw_username})

    return_val["number_of_fixtures"] = number_of_fixtures + 1
    return return_val
示例#51
0
def create_or_update_users_and_groups(domain,
                                      user_specs,
                                      group_memoizer=None,
                                      update_progress=None):
    ret = {"errors": [], "rows": []}

    group_memoizer = group_memoizer or GroupMemoizer(domain)
    group_memoizer.load_all()

    current = 0

    can_assign_locations = domain_has_privilege(domain, privileges.LOCATIONS)
    if can_assign_locations:
        location_cache = SiteCodeToLocationCache(domain)

    domain_obj = Domain.get_by_name(domain)
    allowed_group_names = [group.name for group in group_memoizer.groups]
    roles_by_name = {role.name: role for role in UserRole.by_domain(domain)}
    validators = get_user_import_validators(domain_obj, user_specs,
                                            allowed_group_names,
                                            list(roles_by_name))
    try:
        for row in user_specs:
            if update_progress:
                update_progress(current)
                current += 1

            username = row.get('username')
            status_row = {
                'username': username,
                'row': row,
            }

            try:
                for validator in validators:
                    validator(row)
            except UserUploadError as e:
                status_row['flag'] = str(e)
                ret['rows'].append(status_row)
                continue

            data = row.get('data')
            email = row.get('email')
            group_names = list(map(str, row.get('group') or []))
            language = row.get('language')
            name = row.get('name')
            password = row.get('password')
            phone_number = row.get('phone-number')
            uncategorized_data = row.get('uncategorized_data')
            user_id = row.get('user_id')
            location_codes = row.get('location_code') or []
            if location_codes and not isinstance(location_codes, list):
                location_codes = [location_codes]
            # ignore empty
            location_codes = [code for code in location_codes if code]
            role = row.get('role', '')

            try:
                username = normalize_username(str(username),
                                              domain) if username else None
                password = str(password) if password else None

                is_active = spec_value_to_boolean_or_none(row, 'is_active')
                is_account_confirmed = spec_value_to_boolean_or_none(
                    row, 'is_account_confirmed')

                if user_id:
                    user = CommCareUser.get_by_user_id(user_id, domain)
                    if not user:
                        raise UserUploadError(
                            _("User with ID '{user_id}' not found").format(
                                user_id=user_id, domain=domain))

                    if username and user.username != username:
                        raise UserUploadError(
                            _('Changing usernames is not supported: %(username)r to %(new_username)r'
                              ) % {
                                  'username': user.username,
                                  'new_username': username
                              })

                    # note: explicitly not including "None" here because that's the default value if not set.
                    # False means it was set explicitly to that value
                    if is_account_confirmed is False:
                        raise UserUploadError(
                            _(f"You can only set 'Is Account Confirmed' to 'False' on a new User."
                              ))

                    if is_password(password):
                        user.set_password(password)
                    status_row['flag'] = 'updated'
                else:
                    kwargs = {}
                    if is_account_confirmed is not None:
                        kwargs['is_account_confirmed'] = is_account_confirmed
                    user = CommCareUser.create(domain,
                                               username,
                                               password,
                                               commit=False,
                                               **kwargs)
                    status_row['flag'] = 'created'

                if phone_number:
                    user.add_phone_number(_fmt_phone(phone_number),
                                          default=True)
                if name:
                    user.set_full_name(str(name))
                if data:
                    user.user_data.update(data)
                if uncategorized_data:
                    user.user_data.update(uncategorized_data)
                if language:
                    user.language = language
                if email:
                    user.email = email.lower()
                if is_active is not None:
                    user.is_active = is_active

                if can_assign_locations:
                    # Do this here so that we validate the location code before we
                    # save any other information to the user, this way either all of
                    # the user's information is updated, or none of it
                    location_ids = []
                    for code in location_codes:
                        loc = get_location_from_site_code(code, location_cache)
                        location_ids.append(loc.location_id)

                    locations_updated = set(
                        user.assigned_location_ids) != set(location_ids)
                    primary_location_removed = (
                        user.location_id and not location_ids
                        or user.location_id not in location_ids)

                    if primary_location_removed:
                        user.unset_location(commit=False)
                    if locations_updated:
                        user.reset_locations(location_ids, commit=False)

                if role:
                    user.set_role(domain,
                                  roles_by_name[role].get_qualified_id())

                user.save()

                if is_password(password):
                    # Without this line, digest auth doesn't work.
                    # With this line, digest auth works.
                    # Other than that, I'm not sure what's going on
                    # Passing use_primary_db=True because of https://dimagi-dev.atlassian.net/browse/ICDS-465
                    user.get_django_user(
                        use_primary_db=True).check_password(password)

                for group in group_memoizer.by_user_id(user.user_id):
                    if group.name not in group_names:
                        group.remove_user(user)

                for group_name in group_names:
                    group_memoizer.by_name(group_name).add_user(user,
                                                                save=False)

            except (UserUploadError, CouchUser.Inconsistent) as e:
                status_row['flag'] = str(e)

            ret["rows"].append(status_row)
    finally:
        try:
            group_memoizer.save_all()
        except BulkSaveError as e:
            _error_message = (
                "Oops! We were not able to save some of your group changes. "
                "Please make sure no one else is editing your groups "
                "and try again.")
            logging.exception(('BulkSaveError saving groups. '
                               'User saw error message "%s". Errors: %s') %
                              (_error_message, e.errors))
            ret['errors'].append(_error_message)

    return ret
示例#52
0
 def test_username_doesnt_have_domain(self):
     user = create_restore_user(username=normalize_username('withdomain', domain='thedomain'))
     restore_payload = get_registration_xml(user)
     self.assertTrue('thedomain' not in restore_payload)
示例#53
0
def get_username(xml):
    try:
        username = re.search(r'<[Mm]eta>.*<username>(.*)</username>.*</[Mm]eta>', xml).group(1)
        return normalize_username(username)
    except:
        return None
示例#54
0
def _run_fixture_upload(domain, workbook, replace=False, task=None):
    from corehq.apps.users.bulkupload import GroupMemoizer
    return_val = FixtureUploadResult()
    group_memoizer = GroupMemoizer(domain)
    get_location = get_memoized_location_getter(domain)
    data_types = []

    with CouchTransaction() as transaction:
        type_sheets = workbook.get_all_type_sheets()
        total_tables = len(type_sheets)
        return_val.number_of_fixtures = total_tables

        def _update_progress(table_count, item_count, items_in_table):
            if task:
                processed = table_count * 10 + (10 * item_count /
                                                items_in_table)
                DownloadBase.set_progress(task, processed, 10 * total_tables)

        for table_number, table_def in enumerate(type_sheets):
            tag = table_def.table_id
            new_data_type = FixtureDataType(
                domain=domain,
                is_global=table_def.is_global,
                tag=tag,
                fields=table_def.fields,
                item_attributes=table_def.item_attributes)
            try:
                tagged_fdt = FixtureDataType.fixture_tag_exists(domain, tag)
                if tagged_fdt:
                    data_type = tagged_fdt
                # support old usage with 'UID'
                elif table_def.uid:
                    data_type = FixtureDataType.get(table_def.uid)
                else:
                    data_type = new_data_type

                if replace and data_type != new_data_type:
                    data_type.recursive_delete(transaction)
                    data_type = new_data_type

                data_type.fields = table_def.fields
                data_type.item_attributes = table_def.item_attributes
                data_type.is_global = table_def.is_global
                assert data_type.doc_type == FixtureDataType._doc_type
                if data_type.domain != domain:
                    data_type = new_data_type
                    return_val.errors.append(
                        _("'%(UID)s' is not a valid UID. But the new type is created."
                          ) % {'UID': table_def.uid})
                if table_def.delete:
                    data_type.recursive_delete(transaction)
                    continue
            except (ResourceNotFound, KeyError):
                data_type = new_data_type
            transaction.save(data_type)
            data_types.append(data_type)
            data_items = list(workbook.get_data_sheet(data_type.tag))
            items_in_table = len(data_items)
            for sort_key, di in enumerate(data_items):
                _update_progress(table_number, sort_key, items_in_table)
                type_fields = data_type.fields
                item_fields = {}
                for field in type_fields:
                    # if field doesn't have properties
                    if len(field.properties) == 0:
                        item_fields[field.field_name] = FieldList(field_list=[
                            FixtureItemField(
                                # using unicode here, to cast ints, and multi-language strings
                                field_value=six.text_type(di['field'][
                                    field.field_name]),
                                properties={})
                        ])
                    else:
                        field_list = []
                        field_prop_combos = di['field'][field.field_name]
                        prop_combo_len = len(field_prop_combos)
                        prop_dict = di[field.field_name]
                        for x in range(0, prop_combo_len):
                            fix_item_field = FixtureItemField(
                                field_value=six.text_type(
                                    field_prop_combos[x]),
                                properties={
                                    prop: six.text_type(prop_dict[prop][x])
                                    for prop in prop_dict
                                })
                            field_list.append(fix_item_field)
                        item_fields[field.field_name] = FieldList(
                            field_list=field_list)

                item_attributes = di.get('property', {})
                new_data_item = FixtureDataItem(
                    domain=domain,
                    data_type_id=data_type.get_id,
                    fields=item_fields,
                    item_attributes=item_attributes,
                    sort_key=sort_key)
                try:
                    if di['UID'] and not replace:
                        old_data_item = FixtureDataItem.get(di['UID'])
                    else:
                        old_data_item = new_data_item
                        pass
                    old_data_item.fields = item_fields
                    old_data_item.item_attributes = item_attributes
                    if old_data_item.domain != domain \
                            or not old_data_item.data_type_id == data_type.get_id:
                        old_data_item = new_data_item
                        return_val.errors.append(
                            _("'%(UID)s' is not a valid UID. But the new item is created."
                              ) % {'UID': di['UID']})
                    assert old_data_item.doc_type == FixtureDataItem._doc_type
                    if di[DELETE_HEADER] == "Y" or di[DELETE_HEADER] == "y":
                        old_data_item.recursive_delete(transaction)
                        continue
                except (ResourceNotFound, KeyError):
                    old_data_item = new_data_item
                transaction.save(old_data_item)

                old_groups = old_data_item.groups
                for group in old_groups:
                    old_data_item.remove_group(group)
                old_users = old_data_item.users
                for user in old_users:
                    old_data_item.remove_user(user)
                old_locations = old_data_item.locations
                for location in old_locations:
                    old_data_item.remove_location(location)

                for group_name in di.get('group', []):
                    group = group_memoizer.by_name(group_name)
                    if group:
                        old_data_item.add_group(group, transaction=transaction)
                    else:
                        return_val.errors.append(
                            _("Unknown group: '%(name)s'. But the row is successfully added"
                              ) % {'name': group_name})

                for raw_username in di.get('user', []):
                    try:
                        username = normalize_username(str(raw_username),
                                                      domain)
                    except ValidationError:
                        return_val.errors.append(
                            _("Invalid username: '******'. Row is not added")
                            % {'name': raw_username})
                        continue
                    user = CommCareUser.get_by_username(username)
                    if user:
                        old_data_item.add_user(user)
                    else:
                        return_val.errors.append(
                            _("Unknown user: '******'. But the row is successfully added"
                              ) % {'name': raw_username})

                for name in di.get('location', []):
                    location_cache = get_location(name)
                    if location_cache.is_error:
                        return_val.errors.append(location_cache.message)
                    else:
                        old_data_item.add_location(location_cache.location,
                                                   transaction=transaction)

    clear_fixture_quickcache(domain, data_types)
    clear_fixture_cache(domain)
    return return_val
示例#55
0
def create_or_update_users_and_groups(domain, user_specs, group_specs, location_specs, task=None):
    ret = {"errors": [], "rows": []}
    total = len(user_specs) + len(group_specs) + len(location_specs)
    def _set_progress(progress):
        if task is not None:
            DownloadBase.set_progress(task, progress, total)

    group_memoizer = create_or_update_groups(domain, group_specs, log=ret)
    current = len(group_specs)

    usernames = set()
    user_ids = set()
    allowed_groups = set(group_memoizer.groups)
    allowed_group_names = [group.name for group in allowed_groups]
    try:
        for row in user_specs:
            _set_progress(current)
            current += 1
            data, email, group_names, language, name, password, phone_number, user_id, username = (
                row.get(k) for k in sorted(allowed_headers)
            )
            if password:
                password = unicode(password)
            group_names = group_names or []
            try:
                username = normalize_username(str(username), domain)
            except TypeError:
                username = None
            except ValidationError:
                ret['rows'].append({
                    'username': username,
                    'row': row,
                    'flag': _('username cannot contain spaces or symbols'),
                })
                continue
            status_row = {
                'username': raw_username(username) if username else None,
                'row': row,
            }
            if username in usernames or user_id in user_ids:
                status_row['flag'] = 'repeat'
            elif not username and not user_id:
                status_row['flag'] = 'missing-data'
            else:
                try:
                    if username:
                        usernames.add(username)
                    if user_id:
                        user_ids.add(user_id)
                    if user_id:
                        user = CommCareUser.get_by_user_id(user_id, domain)
                    else:
                        user = CommCareUser.get_by_username(username)

                    def is_password(password):
                        if not password:
                            return False
                        for c in password:
                            if c != "*":
                                return True
                        return False

                    if user:
                        if user.domain != domain:
                            raise UserUploadError(_(
                                'User with username %(username)r is '
                                'somehow in domain %(domain)r'
                            ) % {'username': user.username, 'domain': user.domain})
                        if username and user.username != username:
                            user.change_username(username)
                        if is_password(password):
                            user.set_password(password)
                        status_row['flag'] = 'updated'
                    else:
                        if len(raw_username(username)) > CommCareAccountForm.max_len_username:
                            ret['rows'].append({
                                'username': username,
                                'row': row,
                                'flag': _("username cannot contain greater than %d characters" %
                                          CommCareAccountForm.max_len_username)
                            })
                            continue
                        if not is_password(password):
                            raise UserUploadError(_("Cannot create a new user with a blank password"))
                        user = CommCareUser.create(domain, username, password, uuid=user_id or '', commit=False)
                        status_row['flag'] = 'created'
                    if phone_number:
                        user.add_phone_number(_fmt_phone(phone_number), default=True)
                    if name:
                        user.set_full_name(name)
                    if data:
                        user.user_data.update(data)
                    if language:
                        user.language = language
                    if email:
                        user.email = email
                    user.save()
                    if is_password(password):
                        # Without this line, digest auth doesn't work.
                        # With this line, digest auth works.
                        # Other than that, I'm not sure what's going on
                        user.get_django_user().check_password(password)

                    for group_id in Group.by_user(user, wrap=False):
                        group = group_memoizer.get(group_id)
                        if group.name not in group_names:
                            group.remove_user(user, save=False)

                    for group_name in group_names:
                        if group_name not in allowed_group_names:
                            raise UserUploadError(_(
                                "Can't add to group '%s' "
                                "(try adding it to your spreadsheet)"
                            ) % group_name)
                        group_memoizer.by_name(group_name).add_user(user, save=False)

                except (UserUploadError, CouchUser.Inconsistent) as e:
                    status_row['flag'] = '%s' % e

            ret["rows"].append(status_row)
    finally:
        try:
            group_memoizer.save_all()
        except BulkSaveError as e:
            _error_message = (
                "Oops! We were not able to save some of your group changes. "
                "Please make sure no one else is editing your groups "
                "and try again."
            )
            logging.exception((
                'BulkSaveError saving groups. '
                'User saw error message "%s". Errors: %s'
            ) % (_error_message, e.errors))
            ret['errors'].append(_error_message)

    create_or_update_locations(domain, location_specs, log=ret)
    _set_progress(total)
    return ret
示例#56
0
def create_or_update_commcare_users_and_groups(upload_domain,
                                               user_specs,
                                               upload_user,
                                               upload_record_id,
                                               group_memoizer=None,
                                               update_progress=None):
    """"
    Creates and Updates CommCare Users
    For the associated web user username passed, for each CommCareUser
        if corresponding web user is present
            if web user has confirmed account but not a member of domain
                adds them to the domain with same role and primary location as the CommCareUser
            if already a member of domain
                update their role and primary location to be same as that of the CommCareUser
        else creates or updates user invitation
           sets Invitation with the CommCare user's role and primary location
    All changes to users only, are tracked using UserChangeLogger, as an audit trail.
    """
    from corehq.apps.user_importer.helpers import CommCareUserImporter, WebUserImporter

    domain_info_by_domain = {}

    ret = {"errors": [], "rows": []}
    current = 0
    update_deactivate_after_date = EnterpriseMobileWorkerSettings.is_domain_using_custom_deactivation(
        upload_domain)

    for row in user_specs:
        if update_progress:
            update_progress(current)
            current += 1

        username = row.get('username')
        domain = row.get('domain') or upload_domain
        username = normalize_username(str(username),
                                      domain) if username else None
        status_row = {
            'username': username,
            'row': row,
        }

        try:
            domain_info = get_domain_info(domain, upload_domain, user_specs,
                                          domain_info_by_domain,
                                          group_memoizer)

            for validator in domain_info.validators:
                validator(row)
        except UserUploadError as e:
            status_row['flag'] = str(e)
            ret['rows'].append(status_row)
            continue

        data = row.get('data', {})
        email = row.get('email')
        group_names = list(map(str, row.get('group') or []))
        language = row.get('language')
        name = row.get('name')
        password = row.get('password')
        uncategorized_data = row.get('uncategorized_data', {})
        user_id = row.get('user_id')
        location_codes = row.get('location_code',
                                 []) if 'location_code' in row else None
        location_codes = format_location_codes(location_codes)
        role = row.get('role', None)
        profile = row.get('user_profile', None)
        web_user_username = row.get('web_user')
        phone_numbers = row.get('phone-number',
                                []) if 'phone-number' in row else None

        deactivate_after = row.get(
            'deactivate_after', None) if update_deactivate_after_date else None
        if isinstance(deactivate_after, datetime):
            deactivate_after = deactivate_after.strftime("%m-%Y")
            row['deactivate_after'] = deactivate_after

        try:
            password = str(password) if password else None

            is_active = spec_value_to_boolean_or_none(row, 'is_active')
            is_account_confirmed = spec_value_to_boolean_or_none(
                row, 'is_account_confirmed')
            send_account_confirmation_email = spec_value_to_boolean_or_none(
                row, 'send_confirmation_email')
            remove_web_user = spec_value_to_boolean_or_none(
                row, 'remove_web_user')

            user = _get_or_create_commcare_user(domain, user_id, username,
                                                is_account_confirmed,
                                                web_user_username, password,
                                                upload_user)
            commcare_user_importer = CommCareUserImporter(
                upload_domain,
                domain,
                user,
                upload_user,
                is_new_user=not bool(user_id),
                via=USER_CHANGE_VIA_BULK_IMPORTER,
                upload_record_id=upload_record_id)
            if user_id:
                if is_password(password):
                    commcare_user_importer.update_password(password)
                    # overwrite password in results so we do not save it to the db
                    status_row['row']['password'] = '******'
                status_row['flag'] = 'updated'
            else:
                status_row['flag'] = 'created'

            if phone_numbers is not None:
                phone_numbers = clean_phone_numbers(phone_numbers)
                commcare_user_importer.update_phone_numbers(phone_numbers)

            if name:
                commcare_user_importer.update_name(name)

            commcare_user_importer.update_user_data(data, uncategorized_data,
                                                    profile, domain_info)

            if update_deactivate_after_date:
                commcare_user_importer.update_deactivate_after(
                    deactivate_after)

            if language:
                commcare_user_importer.update_language(language)
            if email:
                commcare_user_importer.update_email(email)
            if is_active is not None:
                commcare_user_importer.update_status(is_active)

            # Do this here so that we validate the location code before we
            # save any other information to the user, this way either all of
            # the user's information is updated, or none of it
            # Do not update location info if the column is not included at all
            if domain_info.can_assign_locations and location_codes is not None:
                commcare_user_importer.update_locations(
                    location_codes, domain_info)

            if role:
                role_qualified_id = domain_info.roles_by_name[role]
                commcare_user_importer.update_role(role_qualified_id)
            elif not commcare_user_importer.logger.is_new_user and 'role' in row:
                commcare_user_importer.update_role('none')

            if web_user_username:
                user.update_metadata({'login_as_user': web_user_username})

            user.save()
            log = commcare_user_importer.save_log()

            if web_user_username:
                check_can_upload_web_users(domain, upload_user)
                web_user = CouchUser.get_by_username(web_user_username)
                if web_user:
                    web_user_importer = WebUserImporter(
                        upload_domain,
                        domain,
                        web_user,
                        upload_user,
                        is_new_user=False,
                        via=USER_CHANGE_VIA_BULK_IMPORTER,
                        upload_record_id=upload_record_id)
                    user_change_logger = web_user_importer.logger
                else:
                    web_user_importer = None
                    user_change_logger = None
                if remove_web_user:
                    remove_web_user_from_domain(domain, web_user, username,
                                                upload_user,
                                                user_change_logger)
                else:
                    check_user_role(username, role)
                    if not web_user and is_account_confirmed:
                        raise UserUploadError(
                            _("You can only set 'Is Account Confirmed' to 'True' on an existing Web User. "
                              f"{web_user_username} is a new username.").
                            format(web_user_username=web_user_username))
                    if web_user and not web_user.is_member_of(
                            domain) and is_account_confirmed:
                        # add confirmed account to domain
                        # role_qualified_id would be be present here as confirmed in check_user_role
                        web_user_importer.add_to_domain(
                            role_qualified_id, user.location_id)
                    elif not web_user or not web_user.is_member_of(domain):
                        create_or_update_web_user_invite(
                            web_user_username,
                            domain,
                            role_qualified_id,
                            upload_user,
                            user.location_id,
                            user_change_logger,
                            send_email=send_account_confirmation_email)
                    elif web_user.is_member_of(domain):
                        # edit existing user in the domain
                        web_user_importer.update_role(role_qualified_id)
                        if location_codes is not None:
                            web_user_importer.update_primary_location(
                                user.location_id)
                        web_user.save()
                if web_user_importer:
                    web_user_importer.save_log()
            if send_account_confirmation_email and not web_user_username:
                send_account_confirmation_if_necessary(user)

            if is_password(password):
                # Without this line, digest auth doesn't work.
                # With this line, digest auth works.
                # Other than that, I'm not sure what's going on
                # Passing use_primary_db=True because of https://dimagi-dev.atlassian.net/browse/ICDS-465
                user.get_django_user(
                    use_primary_db=True).check_password(password)

            group_change_message = commcare_user_importer.update_user_groups(
                domain_info, group_names)

            try:
                domain_info.group_memoizer.save_updated()
            except BulkSaveError as e:
                _error_message = (
                    "Oops! We were not able to save some of your group changes. "
                    "Please make sure no one else is editing your groups "
                    "and try again.")
                logging.exception(('BulkSaveError saving groups. '
                                   'User saw error message "%s". Errors: %s') %
                                  (_error_message, e.errors))
                ret['errors'].append(_error_message)

            if log and group_change_message:
                log.change_messages.update(group_change_message)
                log.save()
            elif group_change_message:
                log = commcare_user_importer.logger.save_only_group_changes(
                    group_change_message)

        except ValidationError as e:
            status_row['flag'] = e.message
        except (UserUploadError, CouchUser.Inconsistent) as e:
            status_row['flag'] = str(e)

        ret["rows"].append(status_row)

    return ret