def test_no_error_email_pattern_for_name_profile(self):
        """Check if does not fail when email pattern is checked with empty values"""

        # None values for name and username
        jrae_uuid = api.add_identity(self.db, 'mls', '*****@*****.**', None,
                                     None)
        api.add_identity(self.db,
                         'mls',
                         '*****@*****.**',
                         None,
                         '',
                         uuid=jrae_uuid)
        api.add_identity(self.db,
                         'mls',
                         '*****@*****.**',
                         None,
                         None,
                         uuid=jrae_uuid)

        # Empty values on name and username fields do not crash
        # when email pattern is check
        self.cmd.autocomplete(['mls', 'its'])

        uids = api.unique_identities(self.db, uuid=jrae_uuid)
        uid = uids[0]

        self.assertEqual(uid.uuid, jrae_uuid)
        self.assertEqual(uid.profile.name, None)
        self.assertEqual(uid.profile.email, '*****@*****.**')
예제 #2
0
    def test_valid_identities_already_exist(self):
        """Check method when an identity already exists but with distinct UUID"""

        # The identity already exists but with a different UUID
        uuid = api.add_identity(self.db, 'unknown', email='*****@*****.**')
        api.add_identity(self.db, source='scm', email='*****@*****.**',
                         name='John Smith', username='******', uuid=uuid)
        api.edit_profile(self.db, uuid, name='John Smith', is_bot=False,
                         country_code='US')

        parser = self.get_parser(datadir('sortinghat_valid.json'))

        code = self.cmd.import_identities(parser)
        self.assertEqual(code, CMD_SUCCESS)

        # Check the contents of the registry
        uids = api.unique_identities(self.db)
        self.assertEqual(len(uids), 2)

        # John Smith
        uid = uids[1]
        self.assertEqual(uid.uuid, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')

        # The profile is updated because a new one was given
        prf = uid.profile
        self.assertEqual(prf.uuid, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')
        self.assertEqual(prf.name, None)
        self.assertEqual(prf.email, '*****@*****.**')
        self.assertEqual(prf.gender, 'male')
        self.assertEqual(prf.gender_acc, 100)
        self.assertEqual(prf.is_bot, True)
        self.assertEqual(prf.country, None)

        ids = self.sort_identities(uid.identities)
        self.assertEqual(len(ids), 3)

        id0 = ids[0]
        self.assertEqual(id0.id, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')
        self.assertEqual(id0.name, None)
        self.assertEqual(id0.email, '*****@*****.**')
        self.assertEqual(id0.username, None)
        self.assertEqual(id0.source, 'unknown')

        id1 = ids[1]
        self.assertEqual(id1.id, '880b3dfcb3a08712e5831bddc3dfe81fc5d7b331')
        self.assertEqual(id1.name, 'John Smith')
        self.assertEqual(id1.email, '*****@*****.**')
        self.assertEqual(id1.username, None)
        self.assertEqual(id1.source, 'scm')

        id2 = ids[2]
        self.assertEqual(id2.id, 'a9b403e150dd4af8953a52a4bb841051e4b705d9')
        self.assertEqual(id2.name, 'John Smith')
        self.assertEqual(id2.email, '*****@*****.**')
        self.assertEqual(id2.username, 'jsmith')
        self.assertEqual(id2.source, 'scm')
예제 #3
0
    def test_valid_identities_already_exist(self):
        """Check method when an identity already exists but with distinct UUID"""

        # The identity already exists but with a different UUID
        uuid = api.add_identity(self.db, 'unknown', email='*****@*****.**')
        api.add_identity(self.db, source='scm', email='*****@*****.**',
                         name='John Smith', username='******', uuid=uuid)
        api.edit_profile(self.db, uuid, name='John Smith', is_bot=False,
                         country_code='US')

        parser = self.get_parser('data/sortinghat_valid.json')

        code = self.cmd.import_identities(parser)
        self.assertEqual(code, CMD_SUCCESS)

        # Check the contents of the registry
        uids = api.unique_identities(self.db)
        self.assertEqual(len(uids), 2)

        # John Smith
        uid = uids[0]
        self.assertEqual(uid.uuid, '23fe3a011190a27a7c5cf6f8925de38ff0994d8d')

        # The profile was not updated because it was already available
        prf = uid.profile
        self.assertEqual(prf.uuid, '23fe3a011190a27a7c5cf6f8925de38ff0994d8d')
        self.assertEqual(prf.name, 'John Smith')
        self.assertEqual(prf.email, None)
        self.assertEqual(prf.is_bot, False)
        self.assertEqual(prf.country_code, 'US')
        self.assertEqual(prf.country.code, 'US')
        self.assertEqual(prf.country.name, 'United States of America')

        ids = self.sort_identities(uid.identities)
        self.assertEqual(len(ids), 3)

        id0 = ids[0]
        self.assertEqual(id0.id, '03e12d00e37fd45593c49a5a5a1652deca4cf302')
        self.assertEqual(id0.name, 'John Smith')
        self.assertEqual(id0.email, '*****@*****.**')
        self.assertEqual(id0.username, 'jsmith')
        self.assertEqual(id0.source, 'scm')

        id1 = ids[1]
        self.assertEqual(id1.id, '23fe3a011190a27a7c5cf6f8925de38ff0994d8d')
        self.assertEqual(id1.name, None)
        self.assertEqual(id1.email, '*****@*****.**')
        self.assertEqual(id1.username, None)
        self.assertEqual(id1.source, 'unknown')

        id2 = ids[2]
        self.assertEqual(id2.id, '75d95d6c8492fd36d24a18bd45d62161e05fbc97')
        self.assertEqual(id2.name, 'John Smith')
        self.assertEqual(id2.email, '*****@*****.**')
        self.assertEqual(id2.username, None)
        self.assertEqual(id2.source, 'scm')
예제 #4
0
    def load_test_dataset(self):
        # Add country
        with self.db.connect() as session:
            # Add a country
            us = Country(code='US', name='United States of America', alpha3='USA')
            session.add(us)

        # Add identity
        api.add_identity(self.db, 'scm', '*****@*****.**',
                         'Jane Roe', 'jroe')
    def load_test_dataset(self):
        # Add country
        with self.db.connect() as session:
            # Add a country
            us = Country(code='US', name='United States of America', alpha3='USA')
            session.add(us)

        # Add identity
        api.add_identity(self.db, 'scm', '*****@*****.**',
                         'Jane Roe', 'jroe')
    def test_valid_identities_already_exist(self):
        """Check method when an identity already exists but with distinct UUID"""

        # The identity already exists but with a different UUID
        uuid = api.add_identity(self.db, 'unknown', email='*****@*****.**')
        api.add_identity(self.db, source='scm', email='*****@*****.**',
                         name='John Smith', username='******', uuid=uuid)
        api.edit_profile(self.db, uuid, name='John Smith', is_bot=False,
                         country_code='US')

        parser = self.get_parser('data/sortinghat_valid.json')

        code = self.cmd.import_identities(parser)
        self.assertEqual(code, CMD_SUCCESS)

        # Check the contents of the registry
        uids = api.unique_identities(self.db)
        self.assertEqual(len(uids), 2)

        # John Smith
        uid = uids[1]
        self.assertEqual(uid.uuid, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')

        # The profile is updated because a new one was given
        prf = uid.profile
        self.assertEqual(prf.uuid, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')
        self.assertEqual(prf.name, None)
        self.assertEqual(prf.email, '*****@*****.**')
        self.assertEqual(prf.is_bot, True)
        self.assertEqual(prf.country, None)

        ids = self.sort_identities(uid.identities)
        self.assertEqual(len(ids), 3)

        id0 = ids[0]
        self.assertEqual(id0.id, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')
        self.assertEqual(id0.name, None)
        self.assertEqual(id0.email, '*****@*****.**')
        self.assertEqual(id0.username, None)
        self.assertEqual(id0.source, 'unknown')

        id1 = ids[1]
        self.assertEqual(id1.id, '880b3dfcb3a08712e5831bddc3dfe81fc5d7b331')
        self.assertEqual(id1.name, 'John Smith')
        self.assertEqual(id1.email, '*****@*****.**')
        self.assertEqual(id1.username, None)
        self.assertEqual(id1.source, 'scm')

        id2 = ids[2]
        self.assertEqual(id2.id, 'a9b403e150dd4af8953a52a4bb841051e4b705d9')
        self.assertEqual(id2.name, 'John Smith')
        self.assertEqual(id2.email, '*****@*****.**')
        self.assertEqual(id2.username, 'jsmith')
        self.assertEqual(id2.source, 'scm')
예제 #7
0
    def _load_test_dataset(self):
        import datetime

        self.db.clear()

        # Add country
        with self.db.connect() as session:
            # Add a country
            us = Country(code='US', name='United States of America', alpha3='USA')
            session.add(us)

        # Add organizations
        api.add_organization(self.db, 'Example')
        api.add_domain(self.db, 'Example', 'example.com', is_top_domain=True)
        api.add_domain(self.db, 'Example', 'example.net', is_top_domain=True)

        api.add_organization(self.db, 'Bitergia')
        api.add_domain(self.db, 'Bitergia', 'bitergia.net', is_top_domain=True)
        api.add_domain(self.db, 'Bitergia', 'bitergia.com', is_top_domain=True)
        api.add_domain(self.db, 'Bitergia', 'api.bitergia.com', is_top_domain=False)
        api.add_domain(self.db, 'Bitergia', 'test.bitergia.com', is_top_domain=False)

        api.add_organization(self.db, 'Unknown')

        # Add John Smith identity
        jsmith_uuid = api.add_identity(self.db, 'scm', '*****@*****.**',
                                       'John Smith', 'jsmith')
        api.add_identity(self.db, 'scm', '*****@*****.**', 'John Smith',
                         uuid=jsmith_uuid)
        api.edit_profile(self.db, jsmith_uuid, email='*****@*****.**', is_bot=True)

        # Add Joe Roe identity
        jroe_uuid = api.add_identity(self.db, 'scm', '*****@*****.**',
                                     'Jane Roe', 'jroe')
        api.add_identity(self.db, 'scm', '*****@*****.**',
                         uuid=jroe_uuid)
        api.add_identity(self.db, 'unknown', '*****@*****.**',
                         uuid=jroe_uuid)
        api.edit_profile(self.db, jroe_uuid, name='Jane Roe', email='*****@*****.**',
                         is_bot=False, country_code='US')

        # Add unique identity, this one won't have neither identities
        # nor enrollments
        api.add_unique_identity(self.db,
                                '0000000000000000000000000000000000000000')

        # Add enrollments
        api.add_enrollment(self.db, jsmith_uuid, 'Example')

        api.add_enrollment(self.db, jroe_uuid, 'Example')
        api.add_enrollment(self.db, jroe_uuid, 'Bitergia',
                           datetime.datetime(1999, 1, 1),
                           datetime.datetime(2000, 1, 1))
        api.add_enrollment(self.db, jroe_uuid, 'Bitergia',
                           datetime.datetime(2006, 1, 1),
                           datetime.datetime(2008, 1, 1))

        # Add blacklist
        api.add_to_matching_blacklist(self.db, '*****@*****.**')
        api.add_to_matching_blacklist(self.db, 'John Smith')
예제 #8
0
    def test_load_identities_no_strict_matching(self):
        """Test to load identities with no strict matching"""

        # First, insert the identity that will match with one
        # from the file
        api.add_identity(self.db, 'unknown', email='jsmith@example')

        code = self.cmd.run('--identities', '--matching', 'default',
                            '--no-strict-matching',
                            datadir('sortinghat_no_strict_valid.json'),
                            '--verbose')
        self.assertEqual(code, CMD_SUCCESS)

        output = sys.stdout.getvalue().strip()
        self.assertEqual(output, LOAD_IDENTITIES_NO_STRICT_OUTPUT)
예제 #9
0
    def test_load_identities_no_strict_matching(self):
        """Test to load identities with no strict matching"""

        # First, insert the identity that will match with one
        # from the file
        api.add_identity(self.db, 'unknown', email='jsmith@example')

        code = self.cmd.run('--identities', '--matching', 'default',
                            '--no-strict-matching',
                            datadir('sortinghat_no_strict_valid.json'),
                            '--verbose')
        self.assertEqual(code, CMD_SUCCESS)

        output = sys.stdout.getvalue().strip()
        self.assertEqual(output, LOAD_IDENTITIES_NO_STRICT_OUTPUT)
예제 #10
0
    def test_multiple_top_domains(self):
        """Check if it choose the right domain when multiple top domains are available"""

        api.add_identity(self.db, 'scm', '*****@*****.**')

        code = self.cmd.affiliate()
        self.assertEqual(code, CMD_SUCCESS)

        output = sys.stdout.getvalue().strip()
        self.assertEqual(output, AFFILIATE_OUTPUT_ALT)

        wrn = sys.stderr.getvalue().strip()
        self.assertEqual(wrn,
                         MULTIPLE_DOMAIN_WARNING % {'subdomain' : 'it.u.example.com',
                                                    'domain' : 'u.example.com'})
예제 #11
0
    def test_matching_no_strict(self):
        """Check if identities with no strict matching are merged"""

        # First, insert the identity that will match with one
        # from the file
        api.add_identity(self.db, 'unknown', email='jsmith@example')

        parser = self.get_parser(datadir('sortinghat_no_strict_valid.json'))

        code = self.cmd.import_identities(parser, matching='default',
                                          no_strict_matching=True)
        self.assertEqual(code, CMD_SUCCESS)

        # Check whether identities were merged
        uids = api.unique_identities(self.db)
        self.assertEqual(len(uids), 1)
        self.assertEqual(len(uids[0].identities), 2)
    def test_multiple_top_domains(self):
        """Check if it choose the right domain when multiple top domains are available"""

        api.add_identity(self.db, 'scm', '*****@*****.**')

        code = self.cmd.affiliate()
        self.assertEqual(code, CMD_SUCCESS)

        output = sys.stdout.getvalue().strip()
        self.assertEqual(output, AFFILIATE_OUTPUT_ALT)

        wrn = sys.stderr.getvalue().strip()
        self.assertEqual(
            wrn, MULTIPLE_DOMAIN_WARNING % {
                'subdomain': 'it.u.example.com',
                'domain': 'u.example.com'
            })
예제 #13
0
    def load_test_dataset(self):
        # Add identities
        jroe_uuid = api.add_identity(self.db, 'scm', '*****@*****.**',
                                     'Jane', 'jroe')
        api.edit_profile(self.db, jroe_uuid, name="Jane Roe")

        jrae_uuid = api.add_identity(self.db, 'scm', '*****@*****.**',
                                     'Jane', 'jrae')
        api.edit_profile(self.db, jrae_uuid, name="Jane R", gender="unknown")

        jsmith_uuid = api.add_identity(self.db, 'mls', '*****@*****.**',
                                       'John Smith', 'jsmith')
        api.edit_profile(self.db, jsmith_uuid, name="John Smith")

        jdoe_uuid = api.add_identity(self.db, 'scm', '*****@*****.**',
                                     'John D', 'jdoe')
        api.edit_profile(self.db, jdoe_uuid, name="John D")
예제 #14
0
    def test_matching_no_strict(self):
        """Check if identities with no strict matching are merged"""

        # First, insert the identity that will match with one
        # from the file
        api.add_identity(self.db, 'unknown', email='jsmith@example')

        parser = self.get_parser(datadir('sortinghat_no_strict_valid.json'))

        code = self.cmd.import_identities(parser,
                                          matching='default',
                                          no_strict_matching=True)
        self.assertEqual(code, CMD_SUCCESS)

        # Check whether identities were merged
        uids = api.unique_identities(self.db)
        self.assertEqual(len(uids), 1)
        self.assertEqual(len(uids[0].identities), 2)
예제 #15
0
    def __get_sh_ids_cache(self, identity_tuple, backend_name):

        # Convert tuple to the original dict
        identity = dict((x, y) for x, y in identity_tuple)

        if not self.sortinghat:
            raise RuntimeError("Sorting Hat not active during enrich")

        iden = {}
        sh_ids = {"id": None, "uuid": None}

        for field in ['email', 'name', 'username']:
            iden[field] = None
            if field in identity:
                iden[field] = identity[field]

        try:
            # Find the uuid for a given id. A bit hacky in SH yet
            api.add_identity(self.sh_db, backend_name, iden['email'],
                             iden['name'], iden['username'])
        except AlreadyExistsError as ex:
            uuid = ex.uuid
            u = api.unique_identities(self.sh_db, uuid)[0]
            sh_ids['id'] = utils.uuid(backend_name,
                                      email=iden['email'],
                                      name=iden['name'],
                                      username=iden['username'])
            sh_ids['uuid'] = u.uuid
        except WrappedValueError:
            logger.warning("None Identity found %s", backend_name)
            logger.warning(identity)
        except NotFoundError:
            logger.error("Identity not found in Sorting Hat %s", backend_name)
            logger.error(identity)
        except UnicodeEncodeError:
            logger.error("UnicodeEncodeError %s", backend_name)
            logger.error(identity)
        except Exception as ex:
            logger.error("Unknown error adding sorting hat identity %s %s", ex,
                         backend_name)
            logger.error(identity)
            logger.error(ex)

        return sh_ids
예제 #16
0
    def __get_uuid_cache(self, identity_tuple, backend_name):

        # Convert tuple to the original dict
        identity = dict((x, y) for x, y in identity_tuple)

        if not self.sortinghat:
            raise RuntimeError("Sorting Hat not active during enrich")

        iden = {}
        uuid = None

        for field in ['email', 'name', 'username']:
            iden[field] = None
            if field in identity:
                iden[field] = identity[field]

        try:
            # Find the uuid for a given id. A bit hacky in SH yet
            api.add_identity(self.sh_db, backend_name,
                             iden['email'], iden['name'],
                             iden['username'])
        except AlreadyExistsError as ex:
            uuid = ex.uuid
            u = api.unique_identities(self.sh_db, uuid)[0]
            uuid = u.uuid
        except WrappedValueError:
            logger.error("None Identity found")
            logger.error("%s %s" % (identity, uuid))
            uuid = None
        except NotFoundError:
            logger.error("Identity found in Sorting Hat which is not unique")
            logger.error("%s %s" % (identity, uuid))
            uuid = None
        except UnicodeEncodeError:
            logger.error("UnicodeEncodeError")
            logger.error("%s %s" % (identity, uuid))
            uuid = None
        except Exception as ex:
            logger.error("Unknown error adding sorting hat identity.")
            logger.error("%s %s" % (identity, uuid))
            uuid = None
        return uuid
예제 #17
0
    def __get_sh_ids_cache(self, identity_tuple, backend_name):

        # Convert tuple to the original dict
        identity = dict((x, y) for x, y in identity_tuple)

        if not self.sortinghat:
            raise RuntimeError("Sorting Hat not active during enrich")

        iden = {}
        sh_ids = {"id": None, "uuid": None}

        for field in ['email', 'name', 'username']:
            iden[field] = None
            if field in identity:
                iden[field] = identity[field]

        try:
            # Find the uuid for a given id. A bit hacky in SH yet
            api.add_identity(self.sh_db, backend_name,
                             iden['email'], iden['name'],
                             iden['username'])
        except AlreadyExistsError as ex:
            uuid = ex.uuid
            u = api.unique_identities(self.sh_db, uuid)[0]
            sh_ids['id'] = utils.uuid(backend_name, email=iden['email'],
                                      name=iden['name'], username=iden['username'])
            sh_ids['uuid'] = u.uuid
        except WrappedValueError:
            logger.error("None Identity found")
            logger.error(identity)
        except NotFoundError:
            logger.error("Identity not found in Sorting Hat")
            logger.error(identity)
        except UnicodeEncodeError:
            logger.error("UnicodeEncodeError")
            logger.error(identity)
        except Exception as ex:
            logger.error("Unknown error adding sorting hat identity %s", ex)
            logger.error(identity)
            logger.error(ex)

        return sh_ids
예제 #18
0
    def __get_uuid_cache(self, identity_tuple, backend_name):

        # Convert tuple to the original dict
        identity = dict((x, y) for x, y in identity_tuple)

        if not self.sortinghat:
            raise RuntimeError("Sorting Hat not active during enrich")

        iden = {}
        uuid = None

        for field in ['email', 'name', 'username']:
            iden[field] = None
            if field in identity:
                iden[field] = identity[field]

        try:
            # Find the uuid for a given id. A bit hacky in SH yet
            api.add_identity(self.sh_db, backend_name,
                             iden['email'], iden['name'],
                             iden['username'])
        except AlreadyExistsError as ex:
            uuid = ex.uuid
            u = api.unique_identities(self.sh_db, uuid)[0]
            uuid = u.uuid
        except WrappedValueError:
            logger.error("None Identity found")
            logger.error("%s %s" % (identity, uuid))
            uuid = None
        except NotFoundError:
            logger.error("Identity found in Sorting Hat which is not unique")
            logger.error("%s %s" % (identity, uuid))
            uuid = None
        except UnicodeEncodeError:
            logger.error("UnicodeEncodeError")
            logger.error("%s %s" % (identity, uuid))
            uuid = None
        except Exception as ex:
            logger.error("Unknown error adding sorting hat identity.")
            logger.error("%s %s" % (identity, uuid))
            uuid = None
        return uuid
    def test_no_error_email_pattern_for_name_profile(self):
        """Check if does not fail when email pattern is checked with empty values"""

        # None values for name and username
        jrae_uuid = api.add_identity(self.db, 'mls', '*****@*****.**',
                                     None, None)
        api.add_identity(self.db, 'mls', '*****@*****.**',
                         None, '', uuid=jrae_uuid)
        api.add_identity(self.db, 'mls', '*****@*****.**',
                         None, None, uuid=jrae_uuid)

        # Empty values on name and username fields do not crash
        # when email pattern is check
        self.cmd.autocomplete(['mls', 'its'])

        uids = api.unique_identities(self.db, uuid=jrae_uuid)
        uid = uids[0]

        self.assertEqual(uid.uuid, jrae_uuid)
        self.assertEqual(uid.profile.name, None)
        self.assertEqual(uid.profile.email, '*****@*****.**')
예제 #20
0
    def test_no_email_on_name_field(self):
        """Check whether an email address is not set as the name in the profile"""

        # Email value as username
        jrae_uuid = api.add_identity(self.db, 'mls', None,
                                     None, '*****@*****.**')
        api.add_identity(self.db, 'mls', None,
                         '*****@*****.**', None, uuid=jrae_uuid)
        api.add_identity(self.db, 'mls', None,
                         None, 'jrae', uuid=jrae_uuid)

        self.cmd.autocomplete(['mls', 'its'])

        uids = api.unique_identities(self.db, uuid=jrae_uuid)
        uid = uids[0]

        # Email values on name and username fields are ignored
        # when the profile is set
        self.assertEqual(uid.uuid, jrae_uuid)
        self.assertEqual(uid.profile.name, 'jrae')
        self.assertEqual(uid.profile.email, None)
예제 #21
0
    def load_test_dataset(self):
        # Add country
        with self.db.connect() as session:
            # Add a country
            us = Country(code='US', name='United States of America', alpha3='USA')
            session.add(us)

        api.add_unique_identity(self.db, 'John Smith')
        api.add_identity(self.db, 'scm', '*****@*****.**',
                         uuid='John Smith')
        api.add_identity(self.db, 'scm', '*****@*****.**', 'John Smith',
                         uuid='John Smith')
        api.edit_profile(self.db, 'John Smith', name='John Smith', is_bot=False)

        api.add_unique_identity(self.db, 'John Doe')
        api.add_identity(self.db, 'scm', '*****@*****.**',
                         uuid='John Doe')
        api.edit_profile(self.db, 'John Doe', email='*****@*****.**', is_bot=True,
                         country_code='US')

        api.add_organization(self.db, 'Example')
        api.add_enrollment(self.db, 'John Smith', 'Example')
        api.add_enrollment(self.db, 'John Doe', 'Example')

        api.add_organization(self.db, 'Bitergia')
        api.add_enrollment(self.db, 'John Smith', 'Bitergia')
        api.add_enrollment(self.db, 'John Doe', 'Bitergia',
                           datetime.datetime(1999, 1, 1),
                           datetime.datetime(2000, 1, 1))

        api.add_organization(self.db, 'LibreSoft')
예제 #22
0
    def load_test_dataset(self):
        api.add_unique_identity(self.db, 'FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF')
        api.add_identity(self.db, 'scm', '*****@*****.**',
                         'John Smith', uuid='FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF')
        api.add_identity(self.db, 'scm', '*****@*****.**',
                         'John Smith', 'jsmith', uuid='FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF')

        api.add_identity(self.db, 'scm', '*****@*****.**',
                         'John Doe', 'jdoe')
예제 #23
0
    def _load_test_dataset(self):
        api.add_unique_identity(self.db, 'FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF')
        api.add_identity(self.db, 'scm', '*****@*****.**',
                         'John Smith', uuid='FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF')
        api.add_identity(self.db, 'scm', '*****@*****.**',
                         'John Smith', 'jsmith', uuid='FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF')

        api.add_identity(self.db, 'scm', '*****@*****.**',
                         'John Doe', 'jdoe')
    def load_test_dataset(self):
        # Add country
        with self.db.connect() as session:
            # Add a country
            us = Country(code='US',
                         name='United States of America',
                         alpha3='USA')
            session.add(us)

        # Add identities
        api.add_identity(self.db, 'scm', '*****@*****.**', 'Jane Roe',
                         'jroe')

        jsmith_uuid = api.add_identity(self.db, 'mls', '*****@*****.**',
                                       None, None)
        api.add_identity(self.db,
                         'its',
                         '*****@*****.**',
                         None,
                         'jsmith',
                         uuid=jsmith_uuid)
        api.add_identity(self.db,
                         'mls',
                         '*****@*****.**',
                         'John Smith',
                         'jsmith',
                         uuid=jsmith_uuid)

        jdoe_uuid = api.add_identity(self.db, 'scm', '*****@*****.**', None,
                                     'jdoe')
        api.add_identity(self.db,
                         'its',
                         None,
                         'John Doe',
                         None,
                         uuid=jdoe_uuid)
예제 #25
0
    def add_identity(cls, db, identity, backend):
        """ Load and identity list from backend in Sorting Hat """
        uuid = None

        try:
            uuid = api.add_identity(db, backend, identity['email'],
                                    identity['name'], identity['username'])

            logger.debug("New sortinghat identity %s %s,%s,%s ", uuid,
                         identity['username'], identity['name'],
                         identity['email'])

            profile = {
                "name":
                identity['name'] if identity['name'] else identity['username'],
                "email":
                identity['email']
            }

            api.edit_profile(db, uuid, **profile)

        except AlreadyExistsError as ex:
            uuid = ex.eid
        except InvalidValueError as ex:
            logger.warning("Trying to add a None identity. Ignoring it.")
        except UnicodeEncodeError as ex:
            logger.warning("UnicodeEncodeError. Ignoring it. %s %s %s",
                           identity['email'], identity['name'],
                           identity['username'])
        except Exception as ex:
            logger.warning(
                "Unknown exception adding identity. Ignoring it. %s %s %s",
                identity['email'],
                identity['name'],
                identity['username'],
                exc_info=True)

        if 'company' in identity and identity['company'] is not None:
            try:
                api.add_organization(db, identity['company'])
                api.add_enrollment(db, uuid, identity['company'],
                                   datetime(1900, 1, 1), datetime(2100, 1, 1))
            except AlreadyExistsError:
                pass

        return uuid
예제 #26
0
    def add_identity(cls, db, identity, backend):
        """ Load and identity list from backend in Sorting Hat """
        uuid = None

        try:
            uuid = api.add_identity(db, backend, identity['email'],
                                    identity['name'], identity['username'])

            logger.debug("New sortinghat identity %s %s,%s,%s ",
                         uuid, identity['username'], identity['name'], identity['email'])

            profile = {"name": identity['name'] if identity['name'] else identity['username'],
                       "email": identity['email']}

            api.edit_profile(db, uuid, **profile)

        except AlreadyExistsError as ex:
            uuid = ex.uuid
        except WrappedValueError as ex:
            logger.warning("Trying to add a None identity. Ignoring it.")
        except UnicodeEncodeError as ex:
            logger.warning("UnicodeEncodeError. Ignoring it. %s %s %s",
                           identity['email'], identity['name'],
                           identity['username'])
        except Exception as ex:
            logger.warning("Unknown exception adding identity. Ignoring it. %s %s %s",
                           identity['email'], identity['name'],
                           identity['username'])
            traceback.print_exc()

        if 'company' in identity and identity['company'] is not None:
            try:
                api.add_organization(db, identity['company'])
                api.add_enrollment(db, uuid, identity['company'],
                                   datetime(1900, 1, 1),
                                   datetime(2100, 1, 1))
            except AlreadyExistsError:
                pass

        return uuid
    def load_test_dataset(self):
        # Add some domains
        api.add_organization(self.db, 'Example')
        api.add_domain(self.db, 'Example', 'example.com', is_top_domain=True)

        api.add_organization(self.db, 'Example Alt')
        api.add_domain(self.db,
                       'Example Alt',
                       'u.example.com',
                       is_top_domain=True)
        api.add_domain(self.db, 'Example Alt', 'es.u.example.com')
        api.add_domain(self.db, 'Example Alt', 'en.u.example.com')

        api.add_organization(self.db, 'Bitergia')
        api.add_domain(self.db, 'Bitergia', 'bitergia.com')
        api.add_domain(self.db, 'Bitergia', 'bitergia.org')

        api.add_organization(self.db, 'LibreSoft')

        # Add some unique identities
        jsmith_uuid = api.add_identity(self.db, 'scm', '*****@*****.**',
                                       'John Smith', 'jsmith')
        api.add_identity(self.db,
                         'scm',
                         '*****@*****.**',
                         'John Smith',
                         uuid=jsmith_uuid)
        api.add_identity(self.db,
                         'scm',
                         '*****@*****.**',
                         'John Smith',
                         'jsmith',
                         uuid=jsmith_uuid)
        api.add_enrollment(self.db, jsmith_uuid, 'Bitergia')

        # Add John Doe identity
        api.add_identity(self.db, 'unknown', None, 'John Doe', 'jdoe')

        # Add Jane Rae identity
        jroe_uuid = api.add_identity(self.db, 'scm', '*****@*****.**',
                                     'Jane Roe', 'jroe')
        api.add_identity(self.db, 'scm', '*****@*****.**', uuid=jroe_uuid)
        api.add_identity(self.db,
                         'unknown',
                         '*****@*****.**',
                         uuid=jroe_uuid)

        # Add no valid email identity
        api.add_identity(self.db, 'test', 'novalidemail@')
예제 #28
0
    def _load_test_dataset(self):
        # Add some domains
        api.add_organization(self.db, 'Example')
        api.add_domain(self.db, 'Example', 'example.com', is_top_domain=True)

        api.add_organization(self.db, 'Example Alt')
        api.add_domain(self.db, 'Example Alt', 'u.example.com', is_top_domain=True)
        api.add_domain(self.db, 'Example Alt', 'es.u.example.com')
        api.add_domain(self.db, 'Example Alt', 'en.u.example.com')

        api.add_organization(self.db, 'Bitergia')
        api.add_domain(self.db, 'Bitergia', 'bitergia.com')
        api.add_domain(self.db, 'Bitergia', 'bitergia.org')

        api.add_organization(self.db, 'LibreSoft')

        # Add some unique identities
        jsmith_uuid = api.add_identity(self.db, 'scm', '*****@*****.**',
                                       'John Smith', 'jsmith')
        api.add_identity(self.db, 'scm', '*****@*****.**', 'John Smith',
                         uuid=jsmith_uuid)
        api.add_identity(self.db, 'scm', '*****@*****.**', 'John Smith', 'jsmith',
                         uuid=jsmith_uuid)
        api.add_enrollment(self.db, jsmith_uuid, 'Bitergia')

        # Add John Doe identity
        api.add_identity(self.db, 'unknown', None, 'John Doe', 'jdoe')

        # Add Jane Rae identity
        jroe_uuid = api.add_identity(self.db, 'scm', '*****@*****.**',
                                     'Jane Roe', 'jroe')
        api.add_identity(self.db, 'scm', '*****@*****.**',
                         uuid=jroe_uuid)
        api.add_identity(self.db, 'unknown', '*****@*****.**',
                         uuid=jroe_uuid)
예제 #29
0
    def test_retry(self):
        """Test if a name is skipped when a connection error is returned"""

        http_requests = setup_genderize_server()

        # This profile won't be updated due to connection errors
        # In this case, a 502 HTTP error
        uuid = api.add_identity(self.db, 'scm', '*****@*****.**',
                                'Error Name')
        api.edit_profile(self.db, uuid, name="Error Name")

        # Tests
        self.cmd.autogender(api_token='abcdefghi')

        uids = api.unique_identities(self.db)

        prf = uids[0].profile
        self.assertEqual(prf.uuid, '2a9ec221b8dd5d5a85ae0e3276b8b2c3618ee15e')
        self.assertEqual(prf.gender, 'female')
        self.assertEqual(prf.gender_acc, 100)

        # Error Name profile was not updated due to connection errors
        prf = uids[1].profile
        self.assertEqual(prf.uuid, '316b78ff088c2a825defacb802013fa670fccb48')
        self.assertEqual(prf.gender, None)
        self.assertEqual(prf.gender_acc, None)

        # Jane Rae gender is not updated because it was already set
        prf = uids[2].profile
        self.assertEqual(prf.uuid, '3e1eccdb1e52ea56225f419d3e532fe9133c7821')
        self.assertEqual(prf.gender, 'unknown')
        self.assertEqual(prf.gender_acc, 100)

        prf = uids[3].profile
        self.assertEqual(prf.uuid, '539acca35c2e8502951a97d2d5af8b0857440b50')
        self.assertEqual(prf.gender, 'male')
        self.assertEqual(prf.gender_acc, 99)

        prf = uids[4].profile
        self.assertEqual(prf.uuid, 'a39ac334be9f17bfc7f9f21bbb25f389388f8e18')
        self.assertEqual(prf.gender, 'male')
        self.assertEqual(prf.gender_acc, 99)

        expected = [
            {
                'name': ['jane'],
                'apikey': ['abcdefghi']
            },
            {
                'name': ['error'],
                'apikey': ['abcdefghi']
            },
            {
                'name': ['john'],
                'apikey': ['abcdefghi']
            }
        ]

        self.assertEqual(len(http_requests), 8)

        req = http_requests[0]
        self.assertEqual(req.method, 'GET')
        self.assertEqual(req.querystring, expected[0])

        for i in range(1, 7):
            req = http_requests[i]
            self.assertEqual(req.method, 'GET')
            self.assertEqual(req.querystring, expected[1])

        req = http_requests[7]
        self.assertEqual(req.method, 'GET')
        self.assertEqual(req.querystring, expected[2])
예제 #30
0
    def test_match_new_identities(self):
        """Check whether it matches only new identities"""

        parser = self.get_parser(datadir('sortinghat_valid.json'))

        code = self.cmd.import_identities(parser, matching='default')
        self.assertEqual(code, CMD_SUCCESS)

        # Check the contents of the registry
        uids = api.unique_identities(self.db)
        self.assertEqual(len(uids), 2)

        # We add a new identity that matches with some of the
        # already inserted
        jsmith_uuid = api.add_identity(self.db, 'unknown', email='*****@*****.**')

        uids = api.unique_identities(self.db)
        self.assertEqual(len(uids), 3)

        # This file has a new identity, only Jane Roe will match.
        parser = self.get_parser(datadir('sortinghat_valid_updated.json'))

        code = self.cmd.import_identities(parser, matching='default',
                                          match_new=True)
        self.assertEqual(code, CMD_SUCCESS)

        uids = api.unique_identities(self.db)
        self.assertEqual(len(uids), 3)

        # Jane Roe
        uid = uids[0]
        self.assertEqual(uid.uuid, '17ab00ed3825ec2f50483e33c88df223264182ba')

        ids = self.sort_identities(uid.identities)
        self.assertEqual(len(ids), 4)

        self.assertEqual(ids[0].id, '17ab00ed3825ec2f50483e33c88df223264182ba')
        self.assertEqual(ids[1].id, '22d1b20763c6f5822bdda8508957486c547bb9de')
        self.assertEqual(ids[2].id, '322397ed782a798ffd9d0bc7e293df4292fe075d')
        self.assertEqual(ids[3].id, '8ff87accaf518070bdb494b87f4d5a10e7605b47')

        # Now, if we reload again the file but setting 'match_new' to false,
        # the identity that we inserted before will match "John Smith"
        parser = self.get_parser(datadir('sortinghat_valid_updated.json'))

        code = self.cmd.import_identities(parser, matching='default')
        self.assertEqual(code, CMD_SUCCESS)

        uids = api.unique_identities(self.db)
        self.assertEqual(len(uids), 2)

        # John Smith
        uid = uids[1]
        self.assertEqual(uid.uuid, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')

        ids = self.sort_identities(uid.identities)
        self.assertEqual(len(ids), 3)

        self.assertEqual(ids[0].id, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')
        self.assertEqual(ids[0].id, jsmith_uuid)
        self.assertEqual(ids[1].id, '880b3dfcb3a08712e5831bddc3dfe81fc5d7b331')
        self.assertEqual(ids[2].id, 'a9b403e150dd4af8953a52a4bb841051e4b705d9')
예제 #31
0
    def test_match_new_identities(self):
        """Check whether it matches only new identities"""

        parser = self.get_parser('data/sortinghat_valid.json')

        code = self.cmd.import_identities(parser, matching='default')
        self.assertEqual(code, CMD_SUCCESS)

        # Check the contents of the registry
        uids = api.unique_identities(self.db)
        self.assertEqual(len(uids), 2)

        # We add a new identity that matches with some of the
        # already inserted
        jsmith_uuid = api.add_identity(self.db, 'unknown', email='*****@*****.**')

        uids = api.unique_identities(self.db)
        self.assertEqual(len(uids), 3)

        # This file has a new identity, only Jane Roe will match.
        parser = self.get_parser('data/sortinghat_valid_updated.json')

        code = self.cmd.import_identities(parser, matching='default',
                                          match_new=True)
        self.assertEqual(code, CMD_SUCCESS)

        uids = api.unique_identities(self.db)
        self.assertEqual(len(uids), 3)

        # Jane Roe
        uid = uids[2]
        self.assertEqual(uid.uuid, '52e0aa0a14826627e633fd15332988686b730ab3')

        ids = self.sort_identities(uid.identities)
        self.assertEqual(len(ids), 4)

        self.assertEqual(ids[0].id, '52e0aa0a14826627e633fd15332988686b730ab3')
        self.assertEqual(ids[1].id, '684bb801b0ab02a8c0b6867711a994c695cbed4a')
        self.assertEqual(ids[2].id, 'cbfb7bd31d556322c640f5bc7b31d58a12b15904')
        self.assertEqual(ids[3].id, 'fef873c50a48cfc057f7aa19f423f81889a8907f')

        # Now, if we reload again the file but setting 'match_new' to false,
        # the identity that we inserted before will match "John Smith"
        parser = self.get_parser('data/sortinghat_valid_updated.json')

        code = self.cmd.import_identities(parser, matching='default')
        self.assertEqual(code, CMD_SUCCESS)

        uids = api.unique_identities(self.db)
        self.assertEqual(len(uids), 2)

        # John Smith
        uid = uids[0]
        self.assertEqual(uid.uuid, '23fe3a011190a27a7c5cf6f8925de38ff0994d8d')

        ids = self.sort_identities(uid.identities)
        self.assertEqual(len(ids), 3)

        self.assertEqual(ids[0].id, '03e12d00e37fd45593c49a5a5a1652deca4cf302')
        self.assertEqual(ids[1].id, '23fe3a011190a27a7c5cf6f8925de38ff0994d8d')
        self.assertEqual(ids[1].id, jsmith_uuid)
        self.assertEqual(ids[2].id, '75d95d6c8492fd36d24a18bd45d62161e05fbc97')
예제 #32
0
    def test_valid_identities_with_default_matching(self):
        """Check insertion, matching and merging of valid data"""

        # First, insert the identity that will match with one
        # from the file
        api.add_organization(self.db, 'Example')
        uuid = api.add_identity(self.db, 'unknown', email='*****@*****.**')
        api.add_enrollment(self.db, uuid, 'Example',
                           datetime.datetime(2000, 1, 1, 0, 0),
                           datetime.datetime(2100, 1, 1, 0, 0))
        api.edit_profile(self.db,
                         uuid,
                         name='John Smith',
                         is_bot=False,
                         country_code='US')

        parser = self.get_parser('data/sortinghat_valid.json')

        code = self.cmd.import_identities(parser, matching='default')
        self.assertEqual(code, CMD_SUCCESS)

        # Check the contents of the registry
        uids = api.unique_identities(self.db)
        self.assertEqual(len(uids), 2)

        # John Smith
        uid = uids[0]
        self.assertEqual(uid.uuid, '23fe3a011190a27a7c5cf6f8925de38ff0994d8d')

        ids = self.sort_identities(uid.identities)
        self.assertEqual(len(ids), 3)

        # The profile was merged
        prf = uid.profile
        self.assertEqual(prf.uuid, '23fe3a011190a27a7c5cf6f8925de38ff0994d8d')
        self.assertEqual(prf.name, 'John Smith')
        self.assertEqual(prf.email, '*****@*****.**')
        self.assertEqual(prf.is_bot, True)
        self.assertEqual(prf.country_code, 'US')
        self.assertEqual(prf.country.code, 'US')
        self.assertEqual(prf.country.name, 'United States of America')

        id0 = ids[0]
        self.assertEqual(id0.id, '03e12d00e37fd45593c49a5a5a1652deca4cf302')
        self.assertEqual(id0.name, 'John Smith')
        self.assertEqual(id0.email, '*****@*****.**')
        self.assertEqual(id0.username, 'jsmith')
        self.assertEqual(id0.source, 'scm')

        id1 = ids[1]
        self.assertEqual(id1.id, '23fe3a011190a27a7c5cf6f8925de38ff0994d8d')
        self.assertEqual(id1.name, None)
        self.assertEqual(id1.email, '*****@*****.**')
        self.assertEqual(id1.username, None)
        self.assertEqual(id1.source, 'unknown')

        id2 = ids[2]
        self.assertEqual(id2.id, '75d95d6c8492fd36d24a18bd45d62161e05fbc97')
        self.assertEqual(id2.name, 'John Smith')
        self.assertEqual(id2.email, '*****@*****.**')
        self.assertEqual(id2.username, None)
        self.assertEqual(id2.source, 'scm')

        # Enrollments were merged
        enrollments = api.enrollments(self.db, uid.uuid)
        self.assertEqual(len(enrollments), 1)

        rol0 = enrollments[0]
        self.assertEqual(rol0.organization.name, 'Example')
        self.assertEqual(rol0.start, datetime.datetime(2000, 1, 1, 0, 0))
        self.assertEqual(rol0.end, datetime.datetime(2100, 1, 1, 0, 0))

        # Jane Roe
        uid = uids[1]
        self.assertEqual(uid.uuid, '52e0aa0a14826627e633fd15332988686b730ab3')

        prf = uid.profile
        self.assertEqual(prf.uuid, '52e0aa0a14826627e633fd15332988686b730ab3')
        self.assertEqual(prf.name, 'Jane Roe')
        self.assertEqual(prf.email, '*****@*****.**')
        self.assertEqual(prf.is_bot, False)
        self.assertEqual(prf.country_code, 'US')
        self.assertEqual(prf.country.alpha3, 'USA')
        self.assertEqual(prf.country.code, 'US')
        self.assertEqual(prf.country.name, 'United States of America')

        ids = self.sort_identities(uid.identities)
        self.assertEqual(len(ids), 3)

        id0 = ids[0]
        self.assertEqual(id0.id, '52e0aa0a14826627e633fd15332988686b730ab3')
        self.assertEqual(id0.name, 'Jane Roe')
        self.assertEqual(id0.email, '*****@*****.**')
        self.assertEqual(id0.username, 'jroe')
        self.assertEqual(id0.source, 'scm')

        id1 = ids[1]
        self.assertEqual(id1.id, 'cbfb7bd31d556322c640f5bc7b31d58a12b15904')
        self.assertEqual(id1.name, None)
        self.assertEqual(id1.email, '*****@*****.**')
        self.assertEqual(id1.username, None)
        self.assertEqual(id1.source, 'unknown')

        id2 = ids[2]
        self.assertEqual(id2.id, 'fef873c50a48cfc057f7aa19f423f81889a8907f')
        self.assertEqual(id2.name, None)
        self.assertEqual(id2.email, '*****@*****.**')
        self.assertEqual(id2.username, None)
        self.assertEqual(id2.source, 'scm')

        enrollments = api.enrollments(self.db, uid.uuid)
        self.assertEqual(len(enrollments), 3)
예제 #33
0
    def test_autogender_name_not_found(self):
        """Test if no gender is set when a name is not found"""

        http_requests = setup_genderize_server()

        # This name won't be found
        uuid = api.add_identity(self.db, 'scm', '*****@*****.**',
                                'Random Name')
        api.edit_profile(self.db, uuid, name="Random Name")

        self.cmd.autogender(api_token='abcdefghi')

        uids = api.unique_identities(self.db)

        prf = uids[0].profile
        self.assertEqual(prf.uuid, '2a9ec221b8dd5d5a85ae0e3276b8b2c3618ee15e')
        self.assertEqual(prf.gender, 'female')
        self.assertEqual(prf.gender_acc, 100)

        # Jane Rae gender is not updated because it was already set
        prf = uids[1].profile
        self.assertEqual(prf.uuid, '3e1eccdb1e52ea56225f419d3e532fe9133c7821')
        self.assertEqual(prf.gender, 'unknown')
        self.assertEqual(prf.gender_acc, 100)

        prf = uids[2].profile
        self.assertEqual(prf.uuid, '539acca35c2e8502951a97d2d5af8b0857440b50')
        self.assertEqual(prf.gender, 'male')
        self.assertEqual(prf.gender_acc, 99)

        prf = uids[3].profile
        self.assertEqual(prf.uuid, 'a39ac334be9f17bfc7f9f21bbb25f389388f8e18')
        self.assertEqual(prf.gender, 'male')
        self.assertEqual(prf.gender_acc, 99)

        # This name was not found and the gender was not updated
        prf = uids[4].profile
        self.assertEqual(prf.uuid, 'cfa19ae04ce0c70902a31084fc75086b61ccfcf2')
        self.assertEqual(prf.gender, None)
        self.assertEqual(prf.gender_acc, None)

        # Check requests
        expected = [
            {
                'name': ['jane'],
                'apikey': ['abcdefghi']
            },
            {
                'name': ['john'],
                'apikey': ['abcdefghi']
            },
            {
                'name': ['random'],
                'apikey': ['abcdefghi']
            },
        ]

        self.assertEqual(len(http_requests), len(expected))

        for i in range(len(expected)):
            self.assertDictEqual(http_requests[i].querystring, expected[i])
예제 #34
0
    def test_reset(self):
        """Check if stored relationships and enrollments are removed before loading"""

        # These identities will be split and enrollments removed
        uuid = api.add_identity(self.db, 'unknown', email='*****@*****.**')
        api.add_identity(self.db,
                         source='scm',
                         email='*****@*****.**',
                         name='John Smith',
                         username='******',
                         uuid=uuid)

        api.add_organization(self.db, 'LibreSoft')
        api.add_enrollment(self.db, uuid, 'LibreSoft',
                           datetime.datetime(2000, 1, 1, 0, 0),
                           datetime.datetime(2100, 1, 1, 0, 0))

        parser = self.get_parser(datadir('sortinghat_valid.json'))

        code = self.cmd.import_identities(parser, reset=True)
        self.assertEqual(code, CMD_SUCCESS)

        # Check the contents of the registry
        uids = api.unique_identities(self.db)
        self.assertEqual(len(uids), 3)

        # Jane Roe
        uid = uids[0]
        self.assertEqual(uid.uuid, '17ab00ed3825ec2f50483e33c88df223264182ba')

        ids = self.sort_identities(uid.identities)
        self.assertEqual(len(ids), 3)

        id0 = ids[0]
        self.assertEqual(id0.id, '17ab00ed3825ec2f50483e33c88df223264182ba')
        self.assertEqual(id0.name, 'Jane Roe')
        self.assertEqual(id0.email, '*****@*****.**')
        self.assertEqual(id0.username, 'jroe')
        self.assertEqual(id0.source, 'scm')

        enrollments = api.enrollments(self.db, uid.uuid)
        self.assertEqual(len(enrollments), 3)

        # [email protected]
        uid = uids[1]
        self.assertEqual(uid.uuid, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')

        ids = self.sort_identities(uid.identities)
        self.assertEqual(len(ids), 1)

        id0 = ids[0]
        self.assertEqual(id0.id, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')
        self.assertEqual(id0.name, None)
        self.assertEqual(id0.email, '*****@*****.**')
        self.assertEqual(id0.username, None)
        self.assertEqual(id0.source, 'unknown')

        enrollments = api.enrollments(self.db, uid.uuid)
        self.assertEqual(len(enrollments), 0)

        # John Smith
        uid = uids[2]
        self.assertEqual(uid.uuid, 'a9b403e150dd4af8953a52a4bb841051e4b705d9')

        ids = self.sort_identities(uid.identities)
        self.assertEqual(len(ids), 2)

        id1 = ids[0]
        self.assertEqual(id1.id, '880b3dfcb3a08712e5831bddc3dfe81fc5d7b331')
        self.assertEqual(id1.name, 'John Smith')
        self.assertEqual(id1.email, '*****@*****.**')
        self.assertEqual(id1.username, None)
        self.assertEqual(id1.source, 'scm')

        id2 = ids[1]
        self.assertEqual(id2.id, 'a9b403e150dd4af8953a52a4bb841051e4b705d9')
        self.assertEqual(id2.name, 'John Smith')
        self.assertEqual(id2.email, '*****@*****.**')
        self.assertEqual(id2.username, 'jsmith')
        self.assertEqual(id2.source, 'scm')

        enrollments = api.enrollments(self.db, uid.uuid)
        self.assertEqual(len(enrollments), 1)
예제 #35
0
    def test_valid_identities_with_default_matching(self):
        """Check insertion, matching and merging of valid data"""

        # First, insert the identity that will match with one
        # from the file
        api.add_organization(self.db, 'Example')
        uuid = api.add_identity(self.db, 'unknown', email='*****@*****.**')
        api.add_enrollment(self.db, uuid, 'Example',
                           datetime.datetime(2000, 1, 1, 0, 0),
                           datetime.datetime(2100, 1, 1, 0, 0))
        api.edit_profile(self.db,
                         uuid,
                         name='John Smith',
                         is_bot=False,
                         country_code='US')

        parser = self.get_parser(datadir('sortinghat_valid.json'))

        code = self.cmd.import_identities(parser, matching='default')
        self.assertEqual(code, CMD_SUCCESS)

        # Check the contents of the registry
        uids = api.unique_identities(self.db)
        self.assertEqual(len(uids), 2)

        # Jane Roe
        uid = uids[0]
        self.assertEqual(uid.uuid, '17ab00ed3825ec2f50483e33c88df223264182ba')

        prf = uid.profile
        self.assertEqual(prf.uuid, '17ab00ed3825ec2f50483e33c88df223264182ba')
        self.assertEqual(prf.name, 'Jane Roe')
        self.assertEqual(prf.email, '*****@*****.**')
        self.assertEqual(prf.gender, None)
        self.assertEqual(prf.gender_acc, None)
        self.assertEqual(prf.is_bot, False)
        self.assertEqual(prf.country_code, 'US')
        self.assertEqual(prf.country.alpha3, 'USA')
        self.assertEqual(prf.country.code, 'US')
        self.assertEqual(prf.country.name, 'United States of America')

        ids = self.sort_identities(uid.identities)
        self.assertEqual(len(ids), 3)

        id0 = ids[0]
        self.assertEqual(id0.id, '17ab00ed3825ec2f50483e33c88df223264182ba')
        self.assertEqual(id0.name, 'Jane Roe')
        self.assertEqual(id0.email, '*****@*****.**')
        self.assertEqual(id0.username, 'jroe')
        self.assertEqual(id0.source, 'scm')

        id1 = ids[1]
        self.assertEqual(id1.id, '22d1b20763c6f5822bdda8508957486c547bb9de')
        self.assertEqual(id1.name, None)
        self.assertEqual(id1.email, '*****@*****.**')
        self.assertEqual(id1.username, None)
        self.assertEqual(id1.source, 'unknown')

        id2 = ids[2]
        self.assertEqual(id2.id, '322397ed782a798ffd9d0bc7e293df4292fe075d')
        self.assertEqual(id2.name, None)
        self.assertEqual(id2.email, '*****@*****.**')
        self.assertEqual(id2.username, None)
        self.assertEqual(id2.source, 'scm')

        enrollments = api.enrollments(self.db, uid.uuid)
        self.assertEqual(len(enrollments), 3)

        # John Smith
        uid = uids[1]
        self.assertEqual(uid.uuid, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')

        ids = self.sort_identities(uid.identities)
        self.assertEqual(len(ids), 3)

        # The profile was merged
        prf = uid.profile
        self.assertEqual(prf.uuid, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')
        self.assertEqual(prf.name, 'John Smith')
        self.assertEqual(prf.email, '*****@*****.**')
        self.assertEqual(prf.gender, 'male')
        self.assertEqual(prf.gender_acc, 100)
        self.assertEqual(prf.is_bot, True)
        self.assertEqual(prf.country_code, 'US')
        self.assertEqual(prf.country.code, 'US')
        self.assertEqual(prf.country.name, 'United States of America')

        id0 = ids[0]
        self.assertEqual(id0.id, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')
        self.assertEqual(id0.name, None)
        self.assertEqual(id0.email, '*****@*****.**')
        self.assertEqual(id0.username, None)
        self.assertEqual(id0.source, 'unknown')

        id1 = ids[1]
        self.assertEqual(id1.id, '880b3dfcb3a08712e5831bddc3dfe81fc5d7b331')
        self.assertEqual(id1.name, 'John Smith')
        self.assertEqual(id1.email, '*****@*****.**')
        self.assertEqual(id1.username, None)
        self.assertEqual(id1.source, 'scm')

        id2 = ids[2]
        self.assertEqual(id2.id, 'a9b403e150dd4af8953a52a4bb841051e4b705d9')
        self.assertEqual(id2.name, 'John Smith')
        self.assertEqual(id2.email, '*****@*****.**')
        self.assertEqual(id2.username, 'jsmith')
        self.assertEqual(id2.source, 'scm')

        # Enrollments were merged
        enrollments = api.enrollments(self.db, uid.uuid)
        self.assertEqual(len(enrollments), 1)

        rol0 = enrollments[0]
        self.assertEqual(rol0.organization.name, 'Example')
        self.assertEqual(rol0.start, datetime.datetime(2000, 1, 1, 0, 0))
        self.assertEqual(rol0.end, datetime.datetime(2100, 1, 1, 0, 0))
예제 #36
0
    def load_test_dataset(self):
        # Add country
        with self.db.connect() as session:
            # Add a country
            us = Country(code='US', name='United States of America', alpha3='USA')
            session.add(us)

        # Add identities
        api.add_identity(self.db, 'scm', '*****@*****.**',
                         'Jane Roe', 'jroe')

        jsmith_uuid = api.add_identity(self.db, 'mls', '*****@*****.**',
                                       None, None)
        api.add_identity(self.db, 'its', '*****@*****.**',
                         None, 'jsmith', uuid=jsmith_uuid)
        api.add_identity(self.db, 'mls', '*****@*****.**',
                         'John Smith', 'jsmith', uuid=jsmith_uuid)
        api.add_identity(self.db, 'mls', '*****@*****.**',
                         'J S', None, uuid=jsmith_uuid)

        jdoe_uuid = api.add_identity(self.db, 'scm', '*****@*****.**',
                                     None, 'jdoe')
        api.add_identity(self.db, 'its', None,
                         None, 'jdoe', uuid=jdoe_uuid)
예제 #37
0
    def test_valid_identities_with_default_matching(self):
        """Check insertion, matching and merging of valid data"""

        # First, insert the identity that will match with one
        # from the file
        api.add_organization(self.db, 'Example')
        uuid = api.add_identity(self.db, 'unknown', email='*****@*****.**')
        api.add_enrollment(self.db, uuid, 'Example',
                           datetime.datetime(2000, 1, 1, 0, 0),
                           datetime.datetime(2100, 1, 1, 0, 0))
        api.edit_profile(self.db, uuid, name='John Smith', is_bot=False,
                         country_code='US')

        parser = self.get_parser(datadir('sortinghat_valid.json'))

        code = self.cmd.import_identities(parser, matching='default')
        self.assertEqual(code, CMD_SUCCESS)

        # Check the contents of the registry
        uids = api.unique_identities(self.db)
        self.assertEqual(len(uids), 2)

        # Jane Roe
        uid = uids[0]
        self.assertEqual(uid.uuid, '17ab00ed3825ec2f50483e33c88df223264182ba')

        prf = uid.profile
        self.assertEqual(prf.uuid, '17ab00ed3825ec2f50483e33c88df223264182ba')
        self.assertEqual(prf.name, 'Jane Roe')
        self.assertEqual(prf.email, '*****@*****.**')
        self.assertEqual(prf.gender, None)
        self.assertEqual(prf.gender_acc, None)
        self.assertEqual(prf.is_bot, False)
        self.assertEqual(prf.country_code, 'US')
        self.assertEqual(prf.country.alpha3, 'USA')
        self.assertEqual(prf.country.code, 'US')
        self.assertEqual(prf.country.name, 'United States of America')

        ids = self.sort_identities(uid.identities)
        self.assertEqual(len(ids), 3)

        id0 = ids[0]
        self.assertEqual(id0.id, '17ab00ed3825ec2f50483e33c88df223264182ba')
        self.assertEqual(id0.name, 'Jane Roe')
        self.assertEqual(id0.email, '*****@*****.**')
        self.assertEqual(id0.username, 'jroe')
        self.assertEqual(id0.source, 'scm')

        id1 = ids[1]
        self.assertEqual(id1.id, '22d1b20763c6f5822bdda8508957486c547bb9de')
        self.assertEqual(id1.name, None)
        self.assertEqual(id1.email, '*****@*****.**')
        self.assertEqual(id1.username, None)
        self.assertEqual(id1.source, 'unknown')

        id2 = ids[2]
        self.assertEqual(id2.id, '322397ed782a798ffd9d0bc7e293df4292fe075d')
        self.assertEqual(id2.name, None)
        self.assertEqual(id2.email, '*****@*****.**')
        self.assertEqual(id2.username, None)
        self.assertEqual(id2.source, 'scm')

        enrollments = api.enrollments(self.db, uid.uuid)
        self.assertEqual(len(enrollments), 3)

        # John Smith
        uid = uids[1]
        self.assertEqual(uid.uuid, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')

        ids = self.sort_identities(uid.identities)
        self.assertEqual(len(ids), 3)

        # The profile was merged
        prf = uid.profile
        self.assertEqual(prf.uuid, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')
        self.assertEqual(prf.name, 'John Smith')
        self.assertEqual(prf.email, '*****@*****.**')
        self.assertEqual(prf.gender, 'male')
        self.assertEqual(prf.gender_acc, 100)
        self.assertEqual(prf.is_bot, True)
        self.assertEqual(prf.country_code, 'US')
        self.assertEqual(prf.country.code, 'US')
        self.assertEqual(prf.country.name, 'United States of America')

        id0 = ids[0]
        self.assertEqual(id0.id, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')
        self.assertEqual(id0.name, None)
        self.assertEqual(id0.email, '*****@*****.**')
        self.assertEqual(id0.username, None)
        self.assertEqual(id0.source, 'unknown')

        id1 = ids[1]
        self.assertEqual(id1.id, '880b3dfcb3a08712e5831bddc3dfe81fc5d7b331')
        self.assertEqual(id1.name, 'John Smith')
        self.assertEqual(id1.email, '*****@*****.**')
        self.assertEqual(id1.username, None)
        self.assertEqual(id1.source, 'scm')

        id2 = ids[2]
        self.assertEqual(id2.id, 'a9b403e150dd4af8953a52a4bb841051e4b705d9')
        self.assertEqual(id2.name, 'John Smith')
        self.assertEqual(id2.email, '*****@*****.**')
        self.assertEqual(id2.username, 'jsmith')
        self.assertEqual(id2.source, 'scm')

        # Enrollments were merged
        enrollments = api.enrollments(self.db, uid.uuid)
        self.assertEqual(len(enrollments), 1)

        rol0 = enrollments[0]
        self.assertEqual(rol0.organization.name, 'Example')
        self.assertEqual(rol0.start, datetime.datetime(2000, 1, 1, 0, 0))
        self.assertEqual(rol0.end, datetime.datetime(2100, 1, 1, 0, 0))
예제 #38
0
    def test_valid_identities_already_exist(self):
        """Check method when an identity already exists but with distinct UUID"""

        # The identity already exists but with a different UUID
        uuid = api.add_identity(self.db, 'unknown', email='*****@*****.**')
        api.add_identity(self.db,
                         source='scm',
                         email='*****@*****.**',
                         name='John Smith',
                         username='******',
                         uuid=uuid)
        api.edit_profile(self.db,
                         uuid,
                         name='John Smith',
                         is_bot=False,
                         country_code='US')

        parser = self.get_parser('data/sortinghat_valid.json')

        code = self.cmd.import_identities(parser)
        self.assertEqual(code, CMD_SUCCESS)

        # Check the contents of the registry
        uids = api.unique_identities(self.db)
        self.assertEqual(len(uids), 2)

        # John Smith
        uid = uids[0]
        self.assertEqual(uid.uuid, '23fe3a011190a27a7c5cf6f8925de38ff0994d8d')

        # The profile was not updated because it was already available
        prf = uid.profile
        self.assertEqual(prf.uuid, '23fe3a011190a27a7c5cf6f8925de38ff0994d8d')
        self.assertEqual(prf.name, 'John Smith')
        self.assertEqual(prf.email, None)
        self.assertEqual(prf.is_bot, False)
        self.assertEqual(prf.country_code, 'US')
        self.assertEqual(prf.country.code, 'US')
        self.assertEqual(prf.country.name, 'United States of America')

        ids = self.sort_identities(uid.identities)
        self.assertEqual(len(ids), 3)

        id0 = ids[0]
        self.assertEqual(id0.id, '03e12d00e37fd45593c49a5a5a1652deca4cf302')
        self.assertEqual(id0.name, 'John Smith')
        self.assertEqual(id0.email, '*****@*****.**')
        self.assertEqual(id0.username, 'jsmith')
        self.assertEqual(id0.source, 'scm')

        id1 = ids[1]
        self.assertEqual(id1.id, '23fe3a011190a27a7c5cf6f8925de38ff0994d8d')
        self.assertEqual(id1.name, None)
        self.assertEqual(id1.email, '*****@*****.**')
        self.assertEqual(id1.username, None)
        self.assertEqual(id1.source, 'unknown')

        id2 = ids[2]
        self.assertEqual(id2.id, '75d95d6c8492fd36d24a18bd45d62161e05fbc97')
        self.assertEqual(id2.name, 'John Smith')
        self.assertEqual(id2.email, '*****@*****.**')
        self.assertEqual(id2.username, None)
        self.assertEqual(id2.source, 'scm')
예제 #39
0
    def test_match_new_identities(self):
        """Check whether it matches only new identities"""

        parser = self.get_parser('data/sortinghat_valid.json')

        code = self.cmd.import_identities(parser, matching='default')
        self.assertEqual(code, CMD_SUCCESS)

        # Check the contents of the registry
        uids = api.unique_identities(self.db)
        self.assertEqual(len(uids), 2)

        # We add a new identity that matches with some of the
        # already inserted
        jsmith_uuid = api.add_identity(self.db,
                                       'unknown',
                                       email='*****@*****.**')

        uids = api.unique_identities(self.db)
        self.assertEqual(len(uids), 3)

        # This file has a new identity, only Jane Roe will match.
        parser = self.get_parser('data/sortinghat_valid_updated.json')

        code = self.cmd.import_identities(parser,
                                          matching='default',
                                          match_new=True)
        self.assertEqual(code, CMD_SUCCESS)

        uids = api.unique_identities(self.db)
        self.assertEqual(len(uids), 3)

        # Jane Roe
        uid = uids[2]
        self.assertEqual(uid.uuid, '52e0aa0a14826627e633fd15332988686b730ab3')

        ids = self.sort_identities(uid.identities)
        self.assertEqual(len(ids), 4)

        self.assertEqual(ids[0].id, '52e0aa0a14826627e633fd15332988686b730ab3')
        self.assertEqual(ids[1].id, '684bb801b0ab02a8c0b6867711a994c695cbed4a')
        self.assertEqual(ids[2].id, 'cbfb7bd31d556322c640f5bc7b31d58a12b15904')
        self.assertEqual(ids[3].id, 'fef873c50a48cfc057f7aa19f423f81889a8907f')

        # Now, if we reload again the file but setting 'match_new' to false,
        # the identity that we inserted before will match "John Smith"
        parser = self.get_parser('data/sortinghat_valid_updated.json')

        code = self.cmd.import_identities(parser, matching='default')
        self.assertEqual(code, CMD_SUCCESS)

        uids = api.unique_identities(self.db)
        self.assertEqual(len(uids), 2)

        # John Smith
        uid = uids[0]
        self.assertEqual(uid.uuid, '23fe3a011190a27a7c5cf6f8925de38ff0994d8d')

        ids = self.sort_identities(uid.identities)
        self.assertEqual(len(ids), 3)

        self.assertEqual(ids[0].id, '03e12d00e37fd45593c49a5a5a1652deca4cf302')
        self.assertEqual(ids[1].id, '23fe3a011190a27a7c5cf6f8925de38ff0994d8d')
        self.assertEqual(ids[1].id, jsmith_uuid)
        self.assertEqual(ids[2].id, '75d95d6c8492fd36d24a18bd45d62161e05fbc97')
예제 #40
0
    def test_autogender_ignore_name_not_well_formed(self):
        """Test if no gender is set when a name is invalid"""

        http_requests = setup_genderize_server()

        # These names are invalid so they will be ignored
        uuid = api.add_identity(self.db, 'scm', '*****@*****.**',
                                'Random Name')
        api.edit_profile(self.db, uuid, name="r4nd0m")

        uuid = api.add_identity(self.db, 'mls', '*****@*****.**',
                                'Another Random Name')
        api.edit_profile(self.db, uuid, name="ARadomName")

        self.cmd.autogender(api_token='abcdefghi')

        uids = api.unique_identities(self.db)

        prf = uids[0].profile
        self.assertEqual(prf.uuid, '2a9ec221b8dd5d5a85ae0e3276b8b2c3618ee15e')
        self.assertEqual(prf.gender, 'female')
        self.assertEqual(prf.gender_acc, 100)

        # Jane Rae gender is not updated because it was already set
        prf = uids[1].profile
        self.assertEqual(prf.uuid, '3e1eccdb1e52ea56225f419d3e532fe9133c7821')
        self.assertEqual(prf.gender, 'unknown')
        self.assertEqual(prf.gender_acc, 100)

        prf = uids[2].profile
        self.assertEqual(prf.uuid, '539acca35c2e8502951a97d2d5af8b0857440b50')
        self.assertEqual(prf.gender, 'male')
        self.assertEqual(prf.gender_acc, 99)

        prf = uids[3].profile
        self.assertEqual(prf.uuid, 'a39ac334be9f17bfc7f9f21bbb25f389388f8e18')
        self.assertEqual(prf.gender, 'male')
        self.assertEqual(prf.gender_acc, 99)

        # These names were ignored and their profile were not set either
        prf = uids[4].profile
        self.assertEqual(prf.uuid, 'cfa19ae04ce0c70902a31084fc75086b61ccfcf2')
        self.assertEqual(prf.gender, None)
        self.assertEqual(prf.gender_acc, None)

        prf = uids[5].profile
        self.assertEqual(prf.uuid, 'ee48da0af80479e81b846bec13fe238c06772701')
        self.assertEqual(prf.gender, None)
        self.assertEqual(prf.gender_acc, None)

        # Check requests.
        # Only two valid names were checked
        expected = [
            {
                'name': ['jane'],
                'apikey': ['abcdefghi']
            },
            {
                'name': ['john'],
                'apikey': ['abcdefghi']
            },
        ]

        self.assertEqual(len(http_requests), len(expected))

        for i in range(len(expected)):
            self.assertDictEqual(http_requests[i].querystring, expected[i])
예제 #41
0
    def test_valid_identities_with_default_matching(self):
        """Check insertion, matching and merging of valid data"""

        # First, insert the identity that will match with one
        # from the file
        api.add_organization(self.db, 'Example')
        uuid = api.add_identity(self.db, 'unknown', email='*****@*****.**')
        api.add_enrollment(self.db, uuid, 'Example',
                           datetime.datetime(2000, 1, 1, 0, 0),
                           datetime.datetime(2100, 1, 1, 0, 0))
        api.edit_profile(self.db, uuid, name='John Smith', is_bot=False,
                         country_code='US')

        parser = self.get_parser('data/sortinghat_valid.json')

        code = self.cmd.import_identities(parser, matching='default')
        self.assertEqual(code, CMD_SUCCESS)

        # Check the contents of the registry
        uids = api.unique_identities(self.db)
        self.assertEqual(len(uids), 2)

        # John Smith
        uid = uids[0]
        self.assertEqual(uid.uuid, '23fe3a011190a27a7c5cf6f8925de38ff0994d8d')

        ids = self.sort_identities(uid.identities)
        self.assertEqual(len(ids), 3)

        # The profile was merged
        prf = uid.profile
        self.assertEqual(prf.uuid, '23fe3a011190a27a7c5cf6f8925de38ff0994d8d')
        self.assertEqual(prf.name, 'John Smith')
        self.assertEqual(prf.email, '*****@*****.**')
        self.assertEqual(prf.is_bot, True)
        self.assertEqual(prf.country_code, 'US')
        self.assertEqual(prf.country.code, 'US')
        self.assertEqual(prf.country.name, 'United States of America')

        id0 = ids[0]
        self.assertEqual(id0.id, '03e12d00e37fd45593c49a5a5a1652deca4cf302')
        self.assertEqual(id0.name, 'John Smith')
        self.assertEqual(id0.email, '*****@*****.**')
        self.assertEqual(id0.username, 'jsmith')
        self.assertEqual(id0.source, 'scm')

        id1 = ids[1]
        self.assertEqual(id1.id, '23fe3a011190a27a7c5cf6f8925de38ff0994d8d')
        self.assertEqual(id1.name, None)
        self.assertEqual(id1.email, '*****@*****.**')
        self.assertEqual(id1.username, None)
        self.assertEqual(id1.source, 'unknown')

        id2 = ids[2]
        self.assertEqual(id2.id, '75d95d6c8492fd36d24a18bd45d62161e05fbc97')
        self.assertEqual(id2.name, 'John Smith')
        self.assertEqual(id2.email, '*****@*****.**')
        self.assertEqual(id2.username, None)
        self.assertEqual(id2.source, 'scm')

        # Enrollments were merged
        enrollments = api.enrollments(self.db, uid.uuid)
        self.assertEqual(len(enrollments), 1)

        rol0 = enrollments[0]
        self.assertEqual(rol0.organization.name, 'Example')
        self.assertEqual(rol0.start, datetime.datetime(2000, 1, 1, 0, 0))
        self.assertEqual(rol0.end, datetime.datetime(2100, 1, 1, 0, 0))

        # Jane Roe
        uid = uids[1]
        self.assertEqual(uid.uuid, '52e0aa0a14826627e633fd15332988686b730ab3')

        prf = uid.profile
        self.assertEqual(prf.uuid, '52e0aa0a14826627e633fd15332988686b730ab3')
        self.assertEqual(prf.name, 'Jane Roe')
        self.assertEqual(prf.email, '*****@*****.**')
        self.assertEqual(prf.is_bot, False)
        self.assertEqual(prf.country_code, 'US')
        self.assertEqual(prf.country.alpha3, 'USA')
        self.assertEqual(prf.country.code, 'US')
        self.assertEqual(prf.country.name, 'United States of America')

        ids = self.sort_identities(uid.identities)
        self.assertEqual(len(ids), 3)

        id0 = ids[0]
        self.assertEqual(id0.id, '52e0aa0a14826627e633fd15332988686b730ab3')
        self.assertEqual(id0.name, 'Jane Roe')
        self.assertEqual(id0.email, '*****@*****.**')
        self.assertEqual(id0.username, 'jroe')
        self.assertEqual(id0.source, 'scm')

        id1 = ids[1]
        self.assertEqual(id1.id, 'cbfb7bd31d556322c640f5bc7b31d58a12b15904')
        self.assertEqual(id1.name, None)
        self.assertEqual(id1.email, '*****@*****.**')
        self.assertEqual(id1.username, None)
        self.assertEqual(id1.source, 'unknown')

        id2 = ids[2]
        self.assertEqual(id2.id, 'fef873c50a48cfc057f7aa19f423f81889a8907f')
        self.assertEqual(id2.name, None)
        self.assertEqual(id2.email, '*****@*****.**')
        self.assertEqual(id2.username, None)
        self.assertEqual(id2.source, 'scm')

        enrollments = api.enrollments(self.db, uid.uuid)
        self.assertEqual(len(enrollments), 3)
예제 #42
0
 def load_test_dataset(self):
     api.add_identity(self.db, 'scm', '*****@*****.**',
                      'John Smith', 'jsmith')
     api.add_identity(self.db, 'scm', '*****@*****.**',
                      'John Smith')
예제 #43
0
    def add_identities(cls, db, identities, backend):
        """ Load identities list from backend in Sorting Hat """

        merge_identities = False

        logger.info("Adding the identities to SortingHat")
        if not merge_identities:
            logger.info("Not doing identities merge")

        total = 0
        lidentities = len(identities)

        if merge_identities:
            merged_identities = []  # old identities merged into new ones
            blacklist = api.blacklist(db)
            matching = 'email-name'  # Not active
            matcher = create_identity_matcher(matching, blacklist)

        for identity in identities:
            try:
                uuid = api.add_identity(db, backend, identity['email'],
                                        identity['name'], identity['username'])

                logger.debug("New sortinghat identity %s %s,%s,%s (%i/%i)" % \
                            (uuid, identity['username'], identity['name'], identity['email'],
                            total, lidentities))

                total += 1
                if not merge_identities:
                    continue  # Don't do the merge here. Too slow in large projects

                # Time to  merge
                matches = api.match_identities(db, uuid, matcher)

                if len(matches) > 1:
                    u = api.unique_identities(db, uuid)[0]
                    for m in matches:
                        # First add the old uuid to the list of changed by merge uuids
                        if m.uuid not in merged_identities:
                            merged_identities.append(m.uuid)
                        if m.uuid == uuid:
                            continue
                        # Merge matched identity into added identity
                        api.merge_unique_identities(db, m.uuid, u.uuid)
                        # uuid = m.uuid
                        # u = api.unique_identities(db, uuid, backend)[0]
                        # Include all identities related to this uuid
                        # merged_identities.append(m.uuid)

            except AlreadyExistsError as ex:
                uuid = ex.uuid
                continue
            except WrappedValueError as ex:
                logging.warning("Trying to add a None identity. Ignoring it.")
                continue
            except UnicodeEncodeError as ex:
                logging.warning("UnicodeEncodeError. Ignoring it. %s %s %s" % \
                                (identity['email'], identity['name'],
                                identity['username']))
                continue
            except Exception as ex:
                logging.warning("Unknown exception adding identity. Ignoring it. %s %s %s" % \
                                (identity['email'], identity['name'],
                                identity['username']))
                continue

            if 'company' in identity and identity['company'] is not None:
                try:
                    api.add_organization(db, identity['company'])
                    api.add_enrollment(db, uuid, identity['company'],
                                       datetime(1900, 1, 1),
                                       datetime(2100, 1, 1))
                except AlreadyExistsError:
                    pass

        logger.info("Total NEW identities: %i" % (total))

        if merge_identities:
            logger.info("Total NEW identities merged: %i" % \
                        (len(merged_identities)))
            return merged_identities
        else:
            return []
예제 #44
0
    def test_match_new_identities(self):
        """Check whether it matches only new identities"""

        parser = self.get_parser(datadir('sortinghat_valid.json'))

        code = self.cmd.import_identities(parser, matching='default')
        self.assertEqual(code, CMD_SUCCESS)

        # Check the contents of the registry
        uids = api.unique_identities(self.db)
        self.assertEqual(len(uids), 2)

        # We add a new identity that matches with some of the
        # already inserted
        jsmith_uuid = api.add_identity(self.db,
                                       'unknown',
                                       email='*****@*****.**')

        uids = api.unique_identities(self.db)
        self.assertEqual(len(uids), 3)

        # This file has a new identity, only Jane Roe will match.
        parser = self.get_parser(datadir('sortinghat_valid_updated.json'))

        code = self.cmd.import_identities(parser,
                                          matching='default',
                                          match_new=True)
        self.assertEqual(code, CMD_SUCCESS)

        uids = api.unique_identities(self.db)
        self.assertEqual(len(uids), 3)

        # Jane Roe
        uid = uids[0]
        self.assertEqual(uid.uuid, '17ab00ed3825ec2f50483e33c88df223264182ba')

        ids = self.sort_identities(uid.identities)
        self.assertEqual(len(ids), 4)

        self.assertEqual(ids[0].id, '17ab00ed3825ec2f50483e33c88df223264182ba')
        self.assertEqual(ids[1].id, '22d1b20763c6f5822bdda8508957486c547bb9de')
        self.assertEqual(ids[2].id, '322397ed782a798ffd9d0bc7e293df4292fe075d')
        self.assertEqual(ids[3].id, '8ff87accaf518070bdb494b87f4d5a10e7605b47')

        # Now, if we reload again the file but setting 'match_new' to false,
        # the identity that we inserted before will match "John Smith"
        parser = self.get_parser(datadir('sortinghat_valid_updated.json'))

        code = self.cmd.import_identities(parser, matching='default')
        self.assertEqual(code, CMD_SUCCESS)

        uids = api.unique_identities(self.db)
        self.assertEqual(len(uids), 2)

        # John Smith
        uid = uids[1]
        self.assertEqual(uid.uuid, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')

        ids = self.sort_identities(uid.identities)
        self.assertEqual(len(ids), 3)

        self.assertEqual(ids[0].id, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')
        self.assertEqual(ids[0].id, jsmith_uuid)
        self.assertEqual(ids[1].id, '880b3dfcb3a08712e5831bddc3dfe81fc5d7b331')
        self.assertEqual(ids[2].id, 'a9b403e150dd4af8953a52a4bb841051e4b705d9')
예제 #45
0
    def add_identities(cls, db, identities, backend):
        """ Load identities list from backend in Sorting Hat """

        merge_identities = False

        logger.info("Adding the identities to SortingHat")
        if not merge_identities:
            logger.info("Not doing identities merge")

        total = 0
        lidentities = len(identities)


        if merge_identities:
            merged_identities = []  # old identities merged into new ones
            blacklist = api.blacklist(db)
            matching = 'email-name'  # Not active
            matcher = create_identity_matcher(matching, blacklist)

        for identity in identities:
            try:
                uuid = api.add_identity(db, backend, identity['email'],
                                        identity['name'], identity['username'])

                logger.debug("New sortinghat identity %s %s,%s,%s (%i/%i)",
                            uuid, identity['username'], identity['name'], identity['email'],
                            total, lidentities)

                profile = {"name": identity['name'] if identity['name'] else identity['username'],
                           "email": identity['email']}

                api.edit_profile(db, uuid, **profile)

                total += 1
                if not merge_identities:
                    continue  # Don't do the merge here. Too slow in large projects

                # Time to  merge
                matches = api.match_identities(db, uuid, matcher)

                if len(matches) > 1:
                    u = api.unique_identities(db, uuid)[0]
                    for m in matches:
                        # First add the old uuid to the list of changed by merge uuids
                        if m.uuid not in merged_identities:
                            merged_identities.append(m.uuid)
                        if m.uuid == uuid:
                            continue
                        # Merge matched identity into added identity
                        api.merge_unique_identities(db, m.uuid, u.uuid)
                        # uuid = m.uuid
                        # u = api.unique_identities(db, uuid, backend)[0]
                        # Include all identities related to this uuid
                        # merged_identities.append(m.uuid)

            except AlreadyExistsError as ex:
                uuid = ex.uuid
                continue
            except WrappedValueError as ex:
                logging.warning("Trying to add a None identity. Ignoring it.")
                continue
            except UnicodeEncodeError as ex:
                logging.warning("UnicodeEncodeError. Ignoring it. %s %s %s" % \
                                (identity['email'], identity['name'],
                                identity['username']))
                continue
            except Exception as ex:
                logging.warning("Unknown exception adding identity. Ignoring it. %s %s %s" % \
                                (identity['email'], identity['name'],
                                identity['username']))
                traceback.print_exc()
                continue

            if 'company' in identity and identity['company'] is not None:
                try:
                    api.add_organization(db, identity['company'])
                    api.add_enrollment(db, uuid, identity['company'],
                                       datetime(1900, 1, 1),
                                       datetime(2100, 1, 1))
                except AlreadyExistsError:
                    pass

        logger.info("Total NEW identities: %i" % (total))

        if merge_identities:
            logger.info("Total NEW identities merged: %i" % \
                        (len(merged_identities)))
            return merged_identities
        else:
            return []
예제 #46
0
    def load_test_dataset(self):
        # Add some unique identities

        uuid = api.add_identity(self.db, source='scm', email='*****@*****.**', name='John Smith')
        api.add_identity(self.db, source='scm', name='John Smith', uuid=uuid)
        api.add_identity(self.db, source='scm', username='******', uuid=uuid)

        uuid = api.add_identity(self.db, source='alt', name='J. Smith', username='******')
        api.add_identity(self.db, source='alt', name='John Smith', username='******', uuid=uuid)
        api.add_identity(self.db, source='alt', email='jsmith', uuid=uuid)

        uuid = api.add_identity(self.db, source='scm', name='Jane Rae')
        api.add_identity(self.db, source='mls', email='*****@*****.**', name='Jane Rae Doe', uuid=uuid)

        uuid = api.add_identity(self.db, source='scm', name='J. Smith', username='******')
        api.add_identity(self.db, source='scm', username='******', uuid=uuid)
        api.add_identity(self.db, source='mls', username='******', uuid=uuid)
        api.add_identity(self.db, source='mls', email='*****@*****.**', name='Smith. J', uuid=uuid)

        uuid = api.add_identity(self.db, source='mls', email='*****@*****.**', name='Jane Rae Doe')
        api.add_identity(self.db, source='mls', name='jrae', uuid=uuid)

        uuid = api.add_identity(self.db, source='scm', name='jrae')
예제 #47
0
    def test_reset(self):
        """Check if stored relationships and enrollments are removed before loading"""

        # These identities will be split and enrollments removed
        uuid = api.add_identity(self.db, 'unknown', email='*****@*****.**')
        api.add_identity(self.db, source='scm', email='*****@*****.**',
                         name='John Smith', username='******', uuid=uuid)

        api.add_organization(self.db, 'LibreSoft')
        api.add_enrollment(self.db, uuid, 'LibreSoft',
                           datetime.datetime(2000, 1, 1, 0, 0),
                           datetime.datetime(2100, 1, 1, 0, 0))

        parser = self.get_parser(datadir('sortinghat_valid.json'))

        code = self.cmd.import_identities(parser, reset=True)
        self.assertEqual(code, CMD_SUCCESS)

        # Check the contents of the registry
        uids = api.unique_identities(self.db)
        self.assertEqual(len(uids), 3)

        # Jane Roe
        uid = uids[0]
        self.assertEqual(uid.uuid, '17ab00ed3825ec2f50483e33c88df223264182ba')

        ids = self.sort_identities(uid.identities)
        self.assertEqual(len(ids), 3)

        id0 = ids[0]
        self.assertEqual(id0.id, '17ab00ed3825ec2f50483e33c88df223264182ba')
        self.assertEqual(id0.name, 'Jane Roe')
        self.assertEqual(id0.email, '*****@*****.**')
        self.assertEqual(id0.username, 'jroe')
        self.assertEqual(id0.source, 'scm')

        enrollments = api.enrollments(self.db, uid.uuid)
        self.assertEqual(len(enrollments), 3)

        # [email protected]
        uid = uids[1]
        self.assertEqual(uid.uuid, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')

        ids = self.sort_identities(uid.identities)
        self.assertEqual(len(ids), 1)

        id0 = ids[0]
        self.assertEqual(id0.id, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')
        self.assertEqual(id0.name, None)
        self.assertEqual(id0.email, '*****@*****.**')
        self.assertEqual(id0.username, None)
        self.assertEqual(id0.source, 'unknown')

        enrollments = api.enrollments(self.db, uid.uuid)
        self.assertEqual(len(enrollments), 0)

        # John Smith
        uid = uids[2]
        self.assertEqual(uid.uuid, 'a9b403e150dd4af8953a52a4bb841051e4b705d9')

        ids = self.sort_identities(uid.identities)
        self.assertEqual(len(ids), 2)

        id1 = ids[0]
        self.assertEqual(id1.id, '880b3dfcb3a08712e5831bddc3dfe81fc5d7b331')
        self.assertEqual(id1.name, 'John Smith')
        self.assertEqual(id1.email, '*****@*****.**')
        self.assertEqual(id1.username, None)
        self.assertEqual(id1.source, 'scm')

        id2 = ids[1]
        self.assertEqual(id2.id, 'a9b403e150dd4af8953a52a4bb841051e4b705d9')
        self.assertEqual(id2.name, 'John Smith')
        self.assertEqual(id2.email, '*****@*****.**')
        self.assertEqual(id2.username, 'jsmith')
        self.assertEqual(id2.source, 'scm')

        enrollments = api.enrollments(self.db, uid.uuid)
        self.assertEqual(len(enrollments), 1)
예제 #48
0
 def load_test_dataset(self):
     api.add_identity(self.db, 'scm', '*****@*****.**',
                      'John Smith', 'jsmith')
     api.add_identity(self.db, 'scm', '*****@*****.**',
                      'John Smith')
예제 #49
0
    def load_test_dataset(self):
        # Add some unique identities

        uuid = api.add_identity(self.db, source='scm', email='*****@*****.**', name='John Smith')
        api.add_identity(self.db, source='scm', name='John Smith', uuid=uuid)
        api.add_identity(self.db, source='scm', username='******', uuid=uuid)

        uuid = api.add_identity(self.db, source='alt', name='J. Smith', username='******')
        api.add_identity(self.db, source='alt', name='John Smith', username='******', uuid=uuid)
        api.add_identity(self.db, source='alt', email='jsmith', uuid=uuid)

        uuid = api.add_identity(self.db, source='scm', name='Jane Rae')
        api.add_identity(self.db, source='mls', email='*****@*****.**', name='Jane Rae Doe', uuid=uuid)

        uuid = api.add_identity(self.db, source='scm', name='J. Smith', username='******')
        api.add_identity(self.db, source='scm', username='******', uuid=uuid)
        api.add_identity(self.db, source='mls', username='******', uuid=uuid)
        api.add_identity(self.db, source='mls', email='*****@*****.**', name='Smith. J', uuid=uuid)

        uuid = api.add_identity(self.db, source='mls', email='*****@*****.**', name='Jane Rae Doe')
        api.add_identity(self.db, source='mls', name='jrae', uuid=uuid)

        uuid = api.add_identity(self.db, source='scm', name='jrae')