def test_no_error_email_pattern_for_name_profile(self):
"""Check if does not fail when email pattern is checked with empty values"""
# None values for name and username
jrae_uuid = api.add_identity(self.db, 'mls', '*****@*****.**', None,
None)
api.add_identity(self.db,
'mls',
'*****@*****.**',
None,
'',
uuid=jrae_uuid)
api.add_identity(self.db,
'mls',
'*****@*****.**',
None,
None,
uuid=jrae_uuid)
# Empty values on name and username fields do not crash
# when email pattern is check
self.cmd.autocomplete(['mls', 'its'])
uids = api.unique_identities(self.db, uuid=jrae_uuid)
uid = uids[0]
self.assertEqual(uid.uuid, jrae_uuid)
self.assertEqual(uid.profile.name, None)
self.assertEqual(uid.profile.email, '*****@*****.**')
def test_valid_identities_already_exist(self):
"""Check method when an identity already exists but with distinct UUID"""
# The identity already exists but with a different UUID
uuid = api.add_identity(self.db, 'unknown', email='*****@*****.**')
api.add_identity(self.db, source='scm', email='*****@*****.**',
name='John Smith', username='******', uuid=uuid)
api.edit_profile(self.db, uuid, name='John Smith', is_bot=False,
country_code='US')
parser = self.get_parser(datadir('sortinghat_valid.json'))
code = self.cmd.import_identities(parser)
self.assertEqual(code, CMD_SUCCESS)
# Check the contents of the registry
uids = api.unique_identities(self.db)
self.assertEqual(len(uids), 2)
# John Smith
uid = uids[1]
self.assertEqual(uid.uuid, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')
# The profile is updated because a new one was given
prf = uid.profile
self.assertEqual(prf.uuid, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')
self.assertEqual(prf.name, None)
self.assertEqual(prf.email, '*****@*****.**')
self.assertEqual(prf.gender, 'male')
self.assertEqual(prf.gender_acc, 100)
self.assertEqual(prf.is_bot, True)
self.assertEqual(prf.country, None)
ids = self.sort_identities(uid.identities)
self.assertEqual(len(ids), 3)
id0 = ids[0]
self.assertEqual(id0.id, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')
self.assertEqual(id0.name, None)
self.assertEqual(id0.email, '*****@*****.**')
self.assertEqual(id0.username, None)
self.assertEqual(id0.source, 'unknown')
id1 = ids[1]
self.assertEqual(id1.id, '880b3dfcb3a08712e5831bddc3dfe81fc5d7b331')
self.assertEqual(id1.name, 'John Smith')
self.assertEqual(id1.email, '*****@*****.**')
self.assertEqual(id1.username, None)
self.assertEqual(id1.source, 'scm')
id2 = ids[2]
self.assertEqual(id2.id, 'a9b403e150dd4af8953a52a4bb841051e4b705d9')
self.assertEqual(id2.name, 'John Smith')
self.assertEqual(id2.email, '*****@*****.**')
self.assertEqual(id2.username, 'jsmith')
self.assertEqual(id2.source, 'scm')
def test_valid_identities_already_exist(self):
"""Check method when an identity already exists but with distinct UUID"""
# The identity already exists but with a different UUID
uuid = api.add_identity(self.db, 'unknown', email='*****@*****.**')
api.add_identity(self.db, source='scm', email='*****@*****.**',
name='John Smith', username='******', uuid=uuid)
api.edit_profile(self.db, uuid, name='John Smith', is_bot=False,
country_code='US')
parser = self.get_parser('data/sortinghat_valid.json')
code = self.cmd.import_identities(parser)
self.assertEqual(code, CMD_SUCCESS)
# Check the contents of the registry
uids = api.unique_identities(self.db)
self.assertEqual(len(uids), 2)
# John Smith
uid = uids[0]
self.assertEqual(uid.uuid, '23fe3a011190a27a7c5cf6f8925de38ff0994d8d')
# The profile was not updated because it was already available
prf = uid.profile
self.assertEqual(prf.uuid, '23fe3a011190a27a7c5cf6f8925de38ff0994d8d')
self.assertEqual(prf.name, 'John Smith')
self.assertEqual(prf.email, None)
self.assertEqual(prf.is_bot, False)
self.assertEqual(prf.country_code, 'US')
self.assertEqual(prf.country.code, 'US')
self.assertEqual(prf.country.name, 'United States of America')
ids = self.sort_identities(uid.identities)
self.assertEqual(len(ids), 3)
id0 = ids[0]
self.assertEqual(id0.id, '03e12d00e37fd45593c49a5a5a1652deca4cf302')
self.assertEqual(id0.name, 'John Smith')
self.assertEqual(id0.email, '*****@*****.**')
self.assertEqual(id0.username, 'jsmith')
self.assertEqual(id0.source, 'scm')
id1 = ids[1]
self.assertEqual(id1.id, '23fe3a011190a27a7c5cf6f8925de38ff0994d8d')
self.assertEqual(id1.name, None)
self.assertEqual(id1.email, '*****@*****.**')
self.assertEqual(id1.username, None)
self.assertEqual(id1.source, 'unknown')
id2 = ids[2]
self.assertEqual(id2.id, '75d95d6c8492fd36d24a18bd45d62161e05fbc97')
self.assertEqual(id2.name, 'John Smith')
self.assertEqual(id2.email, '*****@*****.**')
self.assertEqual(id2.username, None)
self.assertEqual(id2.source, 'scm')
def load_test_dataset(self):
# Add country
with self.db.connect() as session:
# Add a country
us = Country(code='US', name='United States of America', alpha3='USA')
session.add(us)
# Add identity
api.add_identity(self.db, 'scm', '*****@*****.**',
'Jane Roe', 'jroe')
def load_test_dataset(self):
# Add country
with self.db.connect() as session:
# Add a country
us = Country(code='US', name='United States of America', alpha3='USA')
session.add(us)
# Add identity
api.add_identity(self.db, 'scm', '*****@*****.**',
'Jane Roe', 'jroe')
def test_valid_identities_already_exist(self):
"""Check method when an identity already exists but with distinct UUID"""
# The identity already exists but with a different UUID
uuid = api.add_identity(self.db, 'unknown', email='*****@*****.**')
api.add_identity(self.db, source='scm', email='*****@*****.**',
name='John Smith', username='******', uuid=uuid)
api.edit_profile(self.db, uuid, name='John Smith', is_bot=False,
country_code='US')
parser = self.get_parser('data/sortinghat_valid.json')
code = self.cmd.import_identities(parser)
self.assertEqual(code, CMD_SUCCESS)
# Check the contents of the registry
uids = api.unique_identities(self.db)
self.assertEqual(len(uids), 2)
# John Smith
uid = uids[1]
self.assertEqual(uid.uuid, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')
# The profile is updated because a new one was given
prf = uid.profile
self.assertEqual(prf.uuid, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')
self.assertEqual(prf.name, None)
self.assertEqual(prf.email, '*****@*****.**')
self.assertEqual(prf.is_bot, True)
self.assertEqual(prf.country, None)
ids = self.sort_identities(uid.identities)
self.assertEqual(len(ids), 3)
id0 = ids[0]
self.assertEqual(id0.id, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')
self.assertEqual(id0.name, None)
self.assertEqual(id0.email, '*****@*****.**')
self.assertEqual(id0.username, None)
self.assertEqual(id0.source, 'unknown')
id1 = ids[1]
self.assertEqual(id1.id, '880b3dfcb3a08712e5831bddc3dfe81fc5d7b331')
self.assertEqual(id1.name, 'John Smith')
self.assertEqual(id1.email, '*****@*****.**')
self.assertEqual(id1.username, None)
self.assertEqual(id1.source, 'scm')
id2 = ids[2]
self.assertEqual(id2.id, 'a9b403e150dd4af8953a52a4bb841051e4b705d9')
self.assertEqual(id2.name, 'John Smith')
self.assertEqual(id2.email, '*****@*****.**')
self.assertEqual(id2.username, 'jsmith')
self.assertEqual(id2.source, 'scm')
def _load_test_dataset(self):
import datetime
self.db.clear()
# Add country
with self.db.connect() as session:
# Add a country
us = Country(code='US', name='United States of America', alpha3='USA')
session.add(us)
# Add organizations
api.add_organization(self.db, 'Example')
api.add_domain(self.db, 'Example', 'example.com', is_top_domain=True)
api.add_domain(self.db, 'Example', 'example.net', is_top_domain=True)
api.add_organization(self.db, 'Bitergia')
api.add_domain(self.db, 'Bitergia', 'bitergia.net', is_top_domain=True)
api.add_domain(self.db, 'Bitergia', 'bitergia.com', is_top_domain=True)
api.add_domain(self.db, 'Bitergia', 'api.bitergia.com', is_top_domain=False)
api.add_domain(self.db, 'Bitergia', 'test.bitergia.com', is_top_domain=False)
api.add_organization(self.db, 'Unknown')
# Add John Smith identity
jsmith_uuid = api.add_identity(self.db, 'scm', '*****@*****.**',
'John Smith', 'jsmith')
api.add_identity(self.db, 'scm', '*****@*****.**', 'John Smith',
uuid=jsmith_uuid)
api.edit_profile(self.db, jsmith_uuid, email='*****@*****.**', is_bot=True)
# Add Joe Roe identity
jroe_uuid = api.add_identity(self.db, 'scm', '*****@*****.**',
'Jane Roe', 'jroe')
api.add_identity(self.db, 'scm', '*****@*****.**',
uuid=jroe_uuid)
api.add_identity(self.db, 'unknown', '*****@*****.**',
uuid=jroe_uuid)
api.edit_profile(self.db, jroe_uuid, name='Jane Roe', email='*****@*****.**',
is_bot=False, country_code='US')
# Add unique identity, this one won't have neither identities
# nor enrollments
api.add_unique_identity(self.db,
'0000000000000000000000000000000000000000')
# Add enrollments
api.add_enrollment(self.db, jsmith_uuid, 'Example')
api.add_enrollment(self.db, jroe_uuid, 'Example')
api.add_enrollment(self.db, jroe_uuid, 'Bitergia',
datetime.datetime(1999, 1, 1),
datetime.datetime(2000, 1, 1))
api.add_enrollment(self.db, jroe_uuid, 'Bitergia',
datetime.datetime(2006, 1, 1),
datetime.datetime(2008, 1, 1))
# Add blacklist
api.add_to_matching_blacklist(self.db, '*****@*****.**')
api.add_to_matching_blacklist(self.db, 'John Smith')
def test_load_identities_no_strict_matching(self):
"""Test to load identities with no strict matching"""
# First, insert the identity that will match with one
# from the file
api.add_identity(self.db, 'unknown', email='jsmith@example')
code = self.cmd.run('--identities', '--matching', 'default',
'--no-strict-matching',
datadir('sortinghat_no_strict_valid.json'),
'--verbose')
self.assertEqual(code, CMD_SUCCESS)
output = sys.stdout.getvalue().strip()
self.assertEqual(output, LOAD_IDENTITIES_NO_STRICT_OUTPUT)
def test_load_identities_no_strict_matching(self):
"""Test to load identities with no strict matching"""
# First, insert the identity that will match with one
# from the file
api.add_identity(self.db, 'unknown', email='jsmith@example')
code = self.cmd.run('--identities', '--matching', 'default',
'--no-strict-matching',
datadir('sortinghat_no_strict_valid.json'),
'--verbose')
self.assertEqual(code, CMD_SUCCESS)
output = sys.stdout.getvalue().strip()
self.assertEqual(output, LOAD_IDENTITIES_NO_STRICT_OUTPUT)
def test_multiple_top_domains(self):
"""Check if it choose the right domain when multiple top domains are available"""
api.add_identity(self.db, 'scm', '*****@*****.**')
code = self.cmd.affiliate()
self.assertEqual(code, CMD_SUCCESS)
output = sys.stdout.getvalue().strip()
self.assertEqual(output, AFFILIATE_OUTPUT_ALT)
wrn = sys.stderr.getvalue().strip()
self.assertEqual(wrn,
MULTIPLE_DOMAIN_WARNING % {'subdomain' : 'it.u.example.com',
'domain' : 'u.example.com'})
def test_matching_no_strict(self):
"""Check if identities with no strict matching are merged"""
# First, insert the identity that will match with one
# from the file
api.add_identity(self.db, 'unknown', email='jsmith@example')
parser = self.get_parser(datadir('sortinghat_no_strict_valid.json'))
code = self.cmd.import_identities(parser, matching='default',
no_strict_matching=True)
self.assertEqual(code, CMD_SUCCESS)
# Check whether identities were merged
uids = api.unique_identities(self.db)
self.assertEqual(len(uids), 1)
self.assertEqual(len(uids[0].identities), 2)
def test_multiple_top_domains(self):
"""Check if it choose the right domain when multiple top domains are available"""
api.add_identity(self.db, 'scm', '*****@*****.**')
code = self.cmd.affiliate()
self.assertEqual(code, CMD_SUCCESS)
output = sys.stdout.getvalue().strip()
self.assertEqual(output, AFFILIATE_OUTPUT_ALT)
wrn = sys.stderr.getvalue().strip()
self.assertEqual(
wrn, MULTIPLE_DOMAIN_WARNING % {
'subdomain': 'it.u.example.com',
'domain': 'u.example.com'
})
def load_test_dataset(self):
# Add identities
jroe_uuid = api.add_identity(self.db, 'scm', '*****@*****.**',
'Jane', 'jroe')
api.edit_profile(self.db, jroe_uuid, name="Jane Roe")
jrae_uuid = api.add_identity(self.db, 'scm', '*****@*****.**',
'Jane', 'jrae')
api.edit_profile(self.db, jrae_uuid, name="Jane R", gender="unknown")
jsmith_uuid = api.add_identity(self.db, 'mls', '*****@*****.**',
'John Smith', 'jsmith')
api.edit_profile(self.db, jsmith_uuid, name="John Smith")
jdoe_uuid = api.add_identity(self.db, 'scm', '*****@*****.**',
'John D', 'jdoe')
api.edit_profile(self.db, jdoe_uuid, name="John D")
def test_matching_no_strict(self):
"""Check if identities with no strict matching are merged"""
# First, insert the identity that will match with one
# from the file
api.add_identity(self.db, 'unknown', email='jsmith@example')
parser = self.get_parser(datadir('sortinghat_no_strict_valid.json'))
code = self.cmd.import_identities(parser,
matching='default',
no_strict_matching=True)
self.assertEqual(code, CMD_SUCCESS)
# Check whether identities were merged
uids = api.unique_identities(self.db)
self.assertEqual(len(uids), 1)
self.assertEqual(len(uids[0].identities), 2)
def __get_sh_ids_cache(self, identity_tuple, backend_name):
# Convert tuple to the original dict
identity = dict((x, y) for x, y in identity_tuple)
if not self.sortinghat:
raise RuntimeError("Sorting Hat not active during enrich")
iden = {}
sh_ids = {"id": None, "uuid": None}
for field in ['email', 'name', 'username']:
iden[field] = None
if field in identity:
iden[field] = identity[field]
try:
# Find the uuid for a given id. A bit hacky in SH yet
api.add_identity(self.sh_db, backend_name, iden['email'],
iden['name'], iden['username'])
except AlreadyExistsError as ex:
uuid = ex.uuid
u = api.unique_identities(self.sh_db, uuid)[0]
sh_ids['id'] = utils.uuid(backend_name,
email=iden['email'],
name=iden['name'],
username=iden['username'])
sh_ids['uuid'] = u.uuid
except WrappedValueError:
logger.warning("None Identity found %s", backend_name)
logger.warning(identity)
except NotFoundError:
logger.error("Identity not found in Sorting Hat %s", backend_name)
logger.error(identity)
except UnicodeEncodeError:
logger.error("UnicodeEncodeError %s", backend_name)
logger.error(identity)
except Exception as ex:
logger.error("Unknown error adding sorting hat identity %s %s", ex,
backend_name)
logger.error(identity)
logger.error(ex)
return sh_ids
def __get_uuid_cache(self, identity_tuple, backend_name):
# Convert tuple to the original dict
identity = dict((x, y) for x, y in identity_tuple)
if not self.sortinghat:
raise RuntimeError("Sorting Hat not active during enrich")
iden = {}
uuid = None
for field in ['email', 'name', 'username']:
iden[field] = None
if field in identity:
iden[field] = identity[field]
try:
# Find the uuid for a given id. A bit hacky in SH yet
api.add_identity(self.sh_db, backend_name,
iden['email'], iden['name'],
iden['username'])
except AlreadyExistsError as ex:
uuid = ex.uuid
u = api.unique_identities(self.sh_db, uuid)[0]
uuid = u.uuid
except WrappedValueError:
logger.error("None Identity found")
logger.error("%s %s" % (identity, uuid))
uuid = None
except NotFoundError:
logger.error("Identity found in Sorting Hat which is not unique")
logger.error("%s %s" % (identity, uuid))
uuid = None
except UnicodeEncodeError:
logger.error("UnicodeEncodeError")
logger.error("%s %s" % (identity, uuid))
uuid = None
except Exception as ex:
logger.error("Unknown error adding sorting hat identity.")
logger.error("%s %s" % (identity, uuid))
uuid = None
return uuid
def __get_sh_ids_cache(self, identity_tuple, backend_name):
# Convert tuple to the original dict
identity = dict((x, y) for x, y in identity_tuple)
if not self.sortinghat:
raise RuntimeError("Sorting Hat not active during enrich")
iden = {}
sh_ids = {"id": None, "uuid": None}
for field in ['email', 'name', 'username']:
iden[field] = None
if field in identity:
iden[field] = identity[field]
try:
# Find the uuid for a given id. A bit hacky in SH yet
api.add_identity(self.sh_db, backend_name,
iden['email'], iden['name'],
iden['username'])
except AlreadyExistsError as ex:
uuid = ex.uuid
u = api.unique_identities(self.sh_db, uuid)[0]
sh_ids['id'] = utils.uuid(backend_name, email=iden['email'],
name=iden['name'], username=iden['username'])
sh_ids['uuid'] = u.uuid
except WrappedValueError:
logger.error("None Identity found")
logger.error(identity)
except NotFoundError:
logger.error("Identity not found in Sorting Hat")
logger.error(identity)
except UnicodeEncodeError:
logger.error("UnicodeEncodeError")
logger.error(identity)
except Exception as ex:
logger.error("Unknown error adding sorting hat identity %s", ex)
logger.error(identity)
logger.error(ex)
return sh_ids
def __get_uuid_cache(self, identity_tuple, backend_name):
# Convert tuple to the original dict
identity = dict((x, y) for x, y in identity_tuple)
if not self.sortinghat:
raise RuntimeError("Sorting Hat not active during enrich")
iden = {}
uuid = None
for field in ['email', 'name', 'username']:
iden[field] = None
if field in identity:
iden[field] = identity[field]
try:
# Find the uuid for a given id. A bit hacky in SH yet
api.add_identity(self.sh_db, backend_name,
iden['email'], iden['name'],
iden['username'])
except AlreadyExistsError as ex:
uuid = ex.uuid
u = api.unique_identities(self.sh_db, uuid)[0]
uuid = u.uuid
except WrappedValueError:
logger.error("None Identity found")
logger.error("%s %s" % (identity, uuid))
uuid = None
except NotFoundError:
logger.error("Identity found in Sorting Hat which is not unique")
logger.error("%s %s" % (identity, uuid))
uuid = None
except UnicodeEncodeError:
logger.error("UnicodeEncodeError")
logger.error("%s %s" % (identity, uuid))
uuid = None
except Exception as ex:
logger.error("Unknown error adding sorting hat identity.")
logger.error("%s %s" % (identity, uuid))
uuid = None
return uuid
def test_no_error_email_pattern_for_name_profile(self):
"""Check if does not fail when email pattern is checked with empty values"""
# None values for name and username
jrae_uuid = api.add_identity(self.db, 'mls', '*****@*****.**',
None, None)
api.add_identity(self.db, 'mls', '*****@*****.**',
None, '', uuid=jrae_uuid)
api.add_identity(self.db, 'mls', '*****@*****.**',
None, None, uuid=jrae_uuid)
# Empty values on name and username fields do not crash
# when email pattern is check
self.cmd.autocomplete(['mls', 'its'])
uids = api.unique_identities(self.db, uuid=jrae_uuid)
uid = uids[0]
self.assertEqual(uid.uuid, jrae_uuid)
self.assertEqual(uid.profile.name, None)
self.assertEqual(uid.profile.email, '*****@*****.**')
def test_no_email_on_name_field(self):
"""Check whether an email address is not set as the name in the profile"""
# Email value as username
jrae_uuid = api.add_identity(self.db, 'mls', None,
None, '*****@*****.**')
api.add_identity(self.db, 'mls', None,
'*****@*****.**', None, uuid=jrae_uuid)
api.add_identity(self.db, 'mls', None,
None, 'jrae', uuid=jrae_uuid)
self.cmd.autocomplete(['mls', 'its'])
uids = api.unique_identities(self.db, uuid=jrae_uuid)
uid = uids[0]
# Email values on name and username fields are ignored
# when the profile is set
self.assertEqual(uid.uuid, jrae_uuid)
self.assertEqual(uid.profile.name, 'jrae')
self.assertEqual(uid.profile.email, None)
def load_test_dataset(self):
# Add country
with self.db.connect() as session:
# Add a country
us = Country(code='US', name='United States of America', alpha3='USA')
session.add(us)
api.add_unique_identity(self.db, 'John Smith')
api.add_identity(self.db, 'scm', '*****@*****.**',
uuid='John Smith')
api.add_identity(self.db, 'scm', '*****@*****.**', 'John Smith',
uuid='John Smith')
api.edit_profile(self.db, 'John Smith', name='John Smith', is_bot=False)
api.add_unique_identity(self.db, 'John Doe')
api.add_identity(self.db, 'scm', '*****@*****.**',
uuid='John Doe')
api.edit_profile(self.db, 'John Doe', email='*****@*****.**', is_bot=True,
country_code='US')
api.add_organization(self.db, 'Example')
api.add_enrollment(self.db, 'John Smith', 'Example')
api.add_enrollment(self.db, 'John Doe', 'Example')
api.add_organization(self.db, 'Bitergia')
api.add_enrollment(self.db, 'John Smith', 'Bitergia')
api.add_enrollment(self.db, 'John Doe', 'Bitergia',
datetime.datetime(1999, 1, 1),
datetime.datetime(2000, 1, 1))
api.add_organization(self.db, 'LibreSoft')
def load_test_dataset(self):
api.add_unique_identity(self.db, 'FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF')
api.add_identity(self.db, 'scm', '*****@*****.**',
'John Smith', uuid='FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF')
api.add_identity(self.db, 'scm', '*****@*****.**',
'John Smith', 'jsmith', uuid='FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF')
api.add_identity(self.db, 'scm', '*****@*****.**',
'John Doe', 'jdoe')
def _load_test_dataset(self):
api.add_unique_identity(self.db, 'FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF')
api.add_identity(self.db, 'scm', '*****@*****.**',
'John Smith', uuid='FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF')
api.add_identity(self.db, 'scm', '*****@*****.**',
'John Smith', 'jsmith', uuid='FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF')
api.add_identity(self.db, 'scm', '*****@*****.**',
'John Doe', 'jdoe')
def load_test_dataset(self):
# Add country
with self.db.connect() as session:
# Add a country
us = Country(code='US',
name='United States of America',
alpha3='USA')
session.add(us)
# Add identities
api.add_identity(self.db, 'scm', '*****@*****.**', 'Jane Roe',
'jroe')
jsmith_uuid = api.add_identity(self.db, 'mls', '*****@*****.**',
None, None)
api.add_identity(self.db,
'its',
'*****@*****.**',
None,
'jsmith',
uuid=jsmith_uuid)
api.add_identity(self.db,
'mls',
'*****@*****.**',
'John Smith',
'jsmith',
uuid=jsmith_uuid)
jdoe_uuid = api.add_identity(self.db, 'scm', '*****@*****.**', None,
'jdoe')
api.add_identity(self.db,
'its',
None,
'John Doe',
None,
uuid=jdoe_uuid)
def add_identity(cls, db, identity, backend):
""" Load and identity list from backend in Sorting Hat """
uuid = None
try:
uuid = api.add_identity(db, backend, identity['email'],
identity['name'], identity['username'])
logger.debug("New sortinghat identity %s %s,%s,%s ", uuid,
identity['username'], identity['name'],
identity['email'])
profile = {
"name":
identity['name'] if identity['name'] else identity['username'],
"email":
identity['email']
}
api.edit_profile(db, uuid, **profile)
except AlreadyExistsError as ex:
uuid = ex.eid
except InvalidValueError as ex:
logger.warning("Trying to add a None identity. Ignoring it.")
except UnicodeEncodeError as ex:
logger.warning("UnicodeEncodeError. Ignoring it. %s %s %s",
identity['email'], identity['name'],
identity['username'])
except Exception as ex:
logger.warning(
"Unknown exception adding identity. Ignoring it. %s %s %s",
identity['email'],
identity['name'],
identity['username'],
exc_info=True)
if 'company' in identity and identity['company'] is not None:
try:
api.add_organization(db, identity['company'])
api.add_enrollment(db, uuid, identity['company'],
datetime(1900, 1, 1), datetime(2100, 1, 1))
except AlreadyExistsError:
pass
return uuid
def add_identity(cls, db, identity, backend):
""" Load and identity list from backend in Sorting Hat """
uuid = None
try:
uuid = api.add_identity(db, backend, identity['email'],
identity['name'], identity['username'])
logger.debug("New sortinghat identity %s %s,%s,%s ",
uuid, identity['username'], identity['name'], identity['email'])
profile = {"name": identity['name'] if identity['name'] else identity['username'],
"email": identity['email']}
api.edit_profile(db, uuid, **profile)
except AlreadyExistsError as ex:
uuid = ex.uuid
except WrappedValueError as ex:
logger.warning("Trying to add a None identity. Ignoring it.")
except UnicodeEncodeError as ex:
logger.warning("UnicodeEncodeError. Ignoring it. %s %s %s",
identity['email'], identity['name'],
identity['username'])
except Exception as ex:
logger.warning("Unknown exception adding identity. Ignoring it. %s %s %s",
identity['email'], identity['name'],
identity['username'])
traceback.print_exc()
if 'company' in identity and identity['company'] is not None:
try:
api.add_organization(db, identity['company'])
api.add_enrollment(db, uuid, identity['company'],
datetime(1900, 1, 1),
datetime(2100, 1, 1))
except AlreadyExistsError:
pass
return uuid
def load_test_dataset(self):
# Add some domains
api.add_organization(self.db, 'Example')
api.add_domain(self.db, 'Example', 'example.com', is_top_domain=True)
api.add_organization(self.db, 'Example Alt')
api.add_domain(self.db,
'Example Alt',
'u.example.com',
is_top_domain=True)
api.add_domain(self.db, 'Example Alt', 'es.u.example.com')
api.add_domain(self.db, 'Example Alt', 'en.u.example.com')
api.add_organization(self.db, 'Bitergia')
api.add_domain(self.db, 'Bitergia', 'bitergia.com')
api.add_domain(self.db, 'Bitergia', 'bitergia.org')
api.add_organization(self.db, 'LibreSoft')
# Add some unique identities
jsmith_uuid = api.add_identity(self.db, 'scm', '*****@*****.**',
'John Smith', 'jsmith')
api.add_identity(self.db,
'scm',
'*****@*****.**',
'John Smith',
uuid=jsmith_uuid)
api.add_identity(self.db,
'scm',
'*****@*****.**',
'John Smith',
'jsmith',
uuid=jsmith_uuid)
api.add_enrollment(self.db, jsmith_uuid, 'Bitergia')
# Add John Doe identity
api.add_identity(self.db, 'unknown', None, 'John Doe', 'jdoe')
# Add Jane Rae identity
jroe_uuid = api.add_identity(self.db, 'scm', '*****@*****.**',
'Jane Roe', 'jroe')
api.add_identity(self.db, 'scm', '*****@*****.**', uuid=jroe_uuid)
api.add_identity(self.db,
'unknown',
'*****@*****.**',
uuid=jroe_uuid)
# Add no valid email identity
api.add_identity(self.db, 'test', 'novalidemail@')
def _load_test_dataset(self):
# Add some domains
api.add_organization(self.db, 'Example')
api.add_domain(self.db, 'Example', 'example.com', is_top_domain=True)
api.add_organization(self.db, 'Example Alt')
api.add_domain(self.db, 'Example Alt', 'u.example.com', is_top_domain=True)
api.add_domain(self.db, 'Example Alt', 'es.u.example.com')
api.add_domain(self.db, 'Example Alt', 'en.u.example.com')
api.add_organization(self.db, 'Bitergia')
api.add_domain(self.db, 'Bitergia', 'bitergia.com')
api.add_domain(self.db, 'Bitergia', 'bitergia.org')
api.add_organization(self.db, 'LibreSoft')
# Add some unique identities
jsmith_uuid = api.add_identity(self.db, 'scm', '*****@*****.**',
'John Smith', 'jsmith')
api.add_identity(self.db, 'scm', '*****@*****.**', 'John Smith',
uuid=jsmith_uuid)
api.add_identity(self.db, 'scm', '*****@*****.**', 'John Smith', 'jsmith',
uuid=jsmith_uuid)
api.add_enrollment(self.db, jsmith_uuid, 'Bitergia')
# Add John Doe identity
api.add_identity(self.db, 'unknown', None, 'John Doe', 'jdoe')
# Add Jane Rae identity
jroe_uuid = api.add_identity(self.db, 'scm', '*****@*****.**',
'Jane Roe', 'jroe')
api.add_identity(self.db, 'scm', '*****@*****.**',
uuid=jroe_uuid)
api.add_identity(self.db, 'unknown', '*****@*****.**',
uuid=jroe_uuid)
def test_retry(self):
"""Test if a name is skipped when a connection error is returned"""
http_requests = setup_genderize_server()
# This profile won't be updated due to connection errors
# In this case, a 502 HTTP error
uuid = api.add_identity(self.db, 'scm', '*****@*****.**',
'Error Name')
api.edit_profile(self.db, uuid, name="Error Name")
# Tests
self.cmd.autogender(api_token='abcdefghi')
uids = api.unique_identities(self.db)
prf = uids[0].profile
self.assertEqual(prf.uuid, '2a9ec221b8dd5d5a85ae0e3276b8b2c3618ee15e')
self.assertEqual(prf.gender, 'female')
self.assertEqual(prf.gender_acc, 100)
# Error Name profile was not updated due to connection errors
prf = uids[1].profile
self.assertEqual(prf.uuid, '316b78ff088c2a825defacb802013fa670fccb48')
self.assertEqual(prf.gender, None)
self.assertEqual(prf.gender_acc, None)
# Jane Rae gender is not updated because it was already set
prf = uids[2].profile
self.assertEqual(prf.uuid, '3e1eccdb1e52ea56225f419d3e532fe9133c7821')
self.assertEqual(prf.gender, 'unknown')
self.assertEqual(prf.gender_acc, 100)
prf = uids[3].profile
self.assertEqual(prf.uuid, '539acca35c2e8502951a97d2d5af8b0857440b50')
self.assertEqual(prf.gender, 'male')
self.assertEqual(prf.gender_acc, 99)
prf = uids[4].profile
self.assertEqual(prf.uuid, 'a39ac334be9f17bfc7f9f21bbb25f389388f8e18')
self.assertEqual(prf.gender, 'male')
self.assertEqual(prf.gender_acc, 99)
expected = [
{
'name': ['jane'],
'apikey': ['abcdefghi']
},
{
'name': ['error'],
'apikey': ['abcdefghi']
},
{
'name': ['john'],
'apikey': ['abcdefghi']
}
]
self.assertEqual(len(http_requests), 8)
req = http_requests[0]
self.assertEqual(req.method, 'GET')
self.assertEqual(req.querystring, expected[0])
for i in range(1, 7):
req = http_requests[i]
self.assertEqual(req.method, 'GET')
self.assertEqual(req.querystring, expected[1])
req = http_requests[7]
self.assertEqual(req.method, 'GET')
self.assertEqual(req.querystring, expected[2])
def test_match_new_identities(self):
"""Check whether it matches only new identities"""
parser = self.get_parser(datadir('sortinghat_valid.json'))
code = self.cmd.import_identities(parser, matching='default')
self.assertEqual(code, CMD_SUCCESS)
# Check the contents of the registry
uids = api.unique_identities(self.db)
self.assertEqual(len(uids), 2)
# We add a new identity that matches with some of the
# already inserted
jsmith_uuid = api.add_identity(self.db, 'unknown', email='*****@*****.**')
uids = api.unique_identities(self.db)
self.assertEqual(len(uids), 3)
# This file has a new identity, only Jane Roe will match.
parser = self.get_parser(datadir('sortinghat_valid_updated.json'))
code = self.cmd.import_identities(parser, matching='default',
match_new=True)
self.assertEqual(code, CMD_SUCCESS)
uids = api.unique_identities(self.db)
self.assertEqual(len(uids), 3)
# Jane Roe
uid = uids[0]
self.assertEqual(uid.uuid, '17ab00ed3825ec2f50483e33c88df223264182ba')
ids = self.sort_identities(uid.identities)
self.assertEqual(len(ids), 4)
self.assertEqual(ids[0].id, '17ab00ed3825ec2f50483e33c88df223264182ba')
self.assertEqual(ids[1].id, '22d1b20763c6f5822bdda8508957486c547bb9de')
self.assertEqual(ids[2].id, '322397ed782a798ffd9d0bc7e293df4292fe075d')
self.assertEqual(ids[3].id, '8ff87accaf518070bdb494b87f4d5a10e7605b47')
# Now, if we reload again the file but setting 'match_new' to false,
# the identity that we inserted before will match "John Smith"
parser = self.get_parser(datadir('sortinghat_valid_updated.json'))
code = self.cmd.import_identities(parser, matching='default')
self.assertEqual(code, CMD_SUCCESS)
uids = api.unique_identities(self.db)
self.assertEqual(len(uids), 2)
# John Smith
uid = uids[1]
self.assertEqual(uid.uuid, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')
ids = self.sort_identities(uid.identities)
self.assertEqual(len(ids), 3)
self.assertEqual(ids[0].id, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')
self.assertEqual(ids[0].id, jsmith_uuid)
self.assertEqual(ids[1].id, '880b3dfcb3a08712e5831bddc3dfe81fc5d7b331')
self.assertEqual(ids[2].id, 'a9b403e150dd4af8953a52a4bb841051e4b705d9')
def test_match_new_identities(self):
"""Check whether it matches only new identities"""
parser = self.get_parser('data/sortinghat_valid.json')
code = self.cmd.import_identities(parser, matching='default')
self.assertEqual(code, CMD_SUCCESS)
# Check the contents of the registry
uids = api.unique_identities(self.db)
self.assertEqual(len(uids), 2)
# We add a new identity that matches with some of the
# already inserted
jsmith_uuid = api.add_identity(self.db, 'unknown', email='*****@*****.**')
uids = api.unique_identities(self.db)
self.assertEqual(len(uids), 3)
# This file has a new identity, only Jane Roe will match.
parser = self.get_parser('data/sortinghat_valid_updated.json')
code = self.cmd.import_identities(parser, matching='default',
match_new=True)
self.assertEqual(code, CMD_SUCCESS)
uids = api.unique_identities(self.db)
self.assertEqual(len(uids), 3)
# Jane Roe
uid = uids[2]
self.assertEqual(uid.uuid, '52e0aa0a14826627e633fd15332988686b730ab3')
ids = self.sort_identities(uid.identities)
self.assertEqual(len(ids), 4)
self.assertEqual(ids[0].id, '52e0aa0a14826627e633fd15332988686b730ab3')
self.assertEqual(ids[1].id, '684bb801b0ab02a8c0b6867711a994c695cbed4a')
self.assertEqual(ids[2].id, 'cbfb7bd31d556322c640f5bc7b31d58a12b15904')
self.assertEqual(ids[3].id, 'fef873c50a48cfc057f7aa19f423f81889a8907f')
# Now, if we reload again the file but setting 'match_new' to false,
# the identity that we inserted before will match "John Smith"
parser = self.get_parser('data/sortinghat_valid_updated.json')
code = self.cmd.import_identities(parser, matching='default')
self.assertEqual(code, CMD_SUCCESS)
uids = api.unique_identities(self.db)
self.assertEqual(len(uids), 2)
# John Smith
uid = uids[0]
self.assertEqual(uid.uuid, '23fe3a011190a27a7c5cf6f8925de38ff0994d8d')
ids = self.sort_identities(uid.identities)
self.assertEqual(len(ids), 3)
self.assertEqual(ids[0].id, '03e12d00e37fd45593c49a5a5a1652deca4cf302')
self.assertEqual(ids[1].id, '23fe3a011190a27a7c5cf6f8925de38ff0994d8d')
self.assertEqual(ids[1].id, jsmith_uuid)
self.assertEqual(ids[2].id, '75d95d6c8492fd36d24a18bd45d62161e05fbc97')
def test_valid_identities_with_default_matching(self):
"""Check insertion, matching and merging of valid data"""
# First, insert the identity that will match with one
# from the file
api.add_organization(self.db, 'Example')
uuid = api.add_identity(self.db, 'unknown', email='*****@*****.**')
api.add_enrollment(self.db, uuid, 'Example',
datetime.datetime(2000, 1, 1, 0, 0),
datetime.datetime(2100, 1, 1, 0, 0))
api.edit_profile(self.db,
uuid,
name='John Smith',
is_bot=False,
country_code='US')
parser = self.get_parser('data/sortinghat_valid.json')
code = self.cmd.import_identities(parser, matching='default')
self.assertEqual(code, CMD_SUCCESS)
# Check the contents of the registry
uids = api.unique_identities(self.db)
self.assertEqual(len(uids), 2)
# John Smith
uid = uids[0]
self.assertEqual(uid.uuid, '23fe3a011190a27a7c5cf6f8925de38ff0994d8d')
ids = self.sort_identities(uid.identities)
self.assertEqual(len(ids), 3)
# The profile was merged
prf = uid.profile
self.assertEqual(prf.uuid, '23fe3a011190a27a7c5cf6f8925de38ff0994d8d')
self.assertEqual(prf.name, 'John Smith')
self.assertEqual(prf.email, '*****@*****.**')
self.assertEqual(prf.is_bot, True)
self.assertEqual(prf.country_code, 'US')
self.assertEqual(prf.country.code, 'US')
self.assertEqual(prf.country.name, 'United States of America')
id0 = ids[0]
self.assertEqual(id0.id, '03e12d00e37fd45593c49a5a5a1652deca4cf302')
self.assertEqual(id0.name, 'John Smith')
self.assertEqual(id0.email, '*****@*****.**')
self.assertEqual(id0.username, 'jsmith')
self.assertEqual(id0.source, 'scm')
id1 = ids[1]
self.assertEqual(id1.id, '23fe3a011190a27a7c5cf6f8925de38ff0994d8d')
self.assertEqual(id1.name, None)
self.assertEqual(id1.email, '*****@*****.**')
self.assertEqual(id1.username, None)
self.assertEqual(id1.source, 'unknown')
id2 = ids[2]
self.assertEqual(id2.id, '75d95d6c8492fd36d24a18bd45d62161e05fbc97')
self.assertEqual(id2.name, 'John Smith')
self.assertEqual(id2.email, '*****@*****.**')
self.assertEqual(id2.username, None)
self.assertEqual(id2.source, 'scm')
# Enrollments were merged
enrollments = api.enrollments(self.db, uid.uuid)
self.assertEqual(len(enrollments), 1)
rol0 = enrollments[0]
self.assertEqual(rol0.organization.name, 'Example')
self.assertEqual(rol0.start, datetime.datetime(2000, 1, 1, 0, 0))
self.assertEqual(rol0.end, datetime.datetime(2100, 1, 1, 0, 0))
# Jane Roe
uid = uids[1]
self.assertEqual(uid.uuid, '52e0aa0a14826627e633fd15332988686b730ab3')
prf = uid.profile
self.assertEqual(prf.uuid, '52e0aa0a14826627e633fd15332988686b730ab3')
self.assertEqual(prf.name, 'Jane Roe')
self.assertEqual(prf.email, '*****@*****.**')
self.assertEqual(prf.is_bot, False)
self.assertEqual(prf.country_code, 'US')
self.assertEqual(prf.country.alpha3, 'USA')
self.assertEqual(prf.country.code, 'US')
self.assertEqual(prf.country.name, 'United States of America')
ids = self.sort_identities(uid.identities)
self.assertEqual(len(ids), 3)
id0 = ids[0]
self.assertEqual(id0.id, '52e0aa0a14826627e633fd15332988686b730ab3')
self.assertEqual(id0.name, 'Jane Roe')
self.assertEqual(id0.email, '*****@*****.**')
self.assertEqual(id0.username, 'jroe')
self.assertEqual(id0.source, 'scm')
id1 = ids[1]
self.assertEqual(id1.id, 'cbfb7bd31d556322c640f5bc7b31d58a12b15904')
self.assertEqual(id1.name, None)
self.assertEqual(id1.email, '*****@*****.**')
self.assertEqual(id1.username, None)
self.assertEqual(id1.source, 'unknown')
id2 = ids[2]
self.assertEqual(id2.id, 'fef873c50a48cfc057f7aa19f423f81889a8907f')
self.assertEqual(id2.name, None)
self.assertEqual(id2.email, '*****@*****.**')
self.assertEqual(id2.username, None)
self.assertEqual(id2.source, 'scm')
enrollments = api.enrollments(self.db, uid.uuid)
self.assertEqual(len(enrollments), 3)
def test_autogender_name_not_found(self):
"""Test if no gender is set when a name is not found"""
http_requests = setup_genderize_server()
# This name won't be found
uuid = api.add_identity(self.db, 'scm', '*****@*****.**',
'Random Name')
api.edit_profile(self.db, uuid, name="Random Name")
self.cmd.autogender(api_token='abcdefghi')
uids = api.unique_identities(self.db)
prf = uids[0].profile
self.assertEqual(prf.uuid, '2a9ec221b8dd5d5a85ae0e3276b8b2c3618ee15e')
self.assertEqual(prf.gender, 'female')
self.assertEqual(prf.gender_acc, 100)
# Jane Rae gender is not updated because it was already set
prf = uids[1].profile
self.assertEqual(prf.uuid, '3e1eccdb1e52ea56225f419d3e532fe9133c7821')
self.assertEqual(prf.gender, 'unknown')
self.assertEqual(prf.gender_acc, 100)
prf = uids[2].profile
self.assertEqual(prf.uuid, '539acca35c2e8502951a97d2d5af8b0857440b50')
self.assertEqual(prf.gender, 'male')
self.assertEqual(prf.gender_acc, 99)
prf = uids[3].profile
self.assertEqual(prf.uuid, 'a39ac334be9f17bfc7f9f21bbb25f389388f8e18')
self.assertEqual(prf.gender, 'male')
self.assertEqual(prf.gender_acc, 99)
# This name was not found and the gender was not updated
prf = uids[4].profile
self.assertEqual(prf.uuid, 'cfa19ae04ce0c70902a31084fc75086b61ccfcf2')
self.assertEqual(prf.gender, None)
self.assertEqual(prf.gender_acc, None)
# Check requests
expected = [
{
'name': ['jane'],
'apikey': ['abcdefghi']
},
{
'name': ['john'],
'apikey': ['abcdefghi']
},
{
'name': ['random'],
'apikey': ['abcdefghi']
},
]
self.assertEqual(len(http_requests), len(expected))
for i in range(len(expected)):
self.assertDictEqual(http_requests[i].querystring, expected[i])
def test_reset(self):
"""Check if stored relationships and enrollments are removed before loading"""
# These identities will be split and enrollments removed
uuid = api.add_identity(self.db, 'unknown', email='*****@*****.**')
api.add_identity(self.db,
source='scm',
email='*****@*****.**',
name='John Smith',
username='******',
uuid=uuid)
api.add_organization(self.db, 'LibreSoft')
api.add_enrollment(self.db, uuid, 'LibreSoft',
datetime.datetime(2000, 1, 1, 0, 0),
datetime.datetime(2100, 1, 1, 0, 0))
parser = self.get_parser(datadir('sortinghat_valid.json'))
code = self.cmd.import_identities(parser, reset=True)
self.assertEqual(code, CMD_SUCCESS)
# Check the contents of the registry
uids = api.unique_identities(self.db)
self.assertEqual(len(uids), 3)
# Jane Roe
uid = uids[0]
self.assertEqual(uid.uuid, '17ab00ed3825ec2f50483e33c88df223264182ba')
ids = self.sort_identities(uid.identities)
self.assertEqual(len(ids), 3)
id0 = ids[0]
self.assertEqual(id0.id, '17ab00ed3825ec2f50483e33c88df223264182ba')
self.assertEqual(id0.name, 'Jane Roe')
self.assertEqual(id0.email, '*****@*****.**')
self.assertEqual(id0.username, 'jroe')
self.assertEqual(id0.source, 'scm')
enrollments = api.enrollments(self.db, uid.uuid)
self.assertEqual(len(enrollments), 3)
# [email protected]
uid = uids[1]
self.assertEqual(uid.uuid, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')
ids = self.sort_identities(uid.identities)
self.assertEqual(len(ids), 1)
id0 = ids[0]
self.assertEqual(id0.id, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')
self.assertEqual(id0.name, None)
self.assertEqual(id0.email, '*****@*****.**')
self.assertEqual(id0.username, None)
self.assertEqual(id0.source, 'unknown')
enrollments = api.enrollments(self.db, uid.uuid)
self.assertEqual(len(enrollments), 0)
# John Smith
uid = uids[2]
self.assertEqual(uid.uuid, 'a9b403e150dd4af8953a52a4bb841051e4b705d9')
ids = self.sort_identities(uid.identities)
self.assertEqual(len(ids), 2)
id1 = ids[0]
self.assertEqual(id1.id, '880b3dfcb3a08712e5831bddc3dfe81fc5d7b331')
self.assertEqual(id1.name, 'John Smith')
self.assertEqual(id1.email, '*****@*****.**')
self.assertEqual(id1.username, None)
self.assertEqual(id1.source, 'scm')
id2 = ids[1]
self.assertEqual(id2.id, 'a9b403e150dd4af8953a52a4bb841051e4b705d9')
self.assertEqual(id2.name, 'John Smith')
self.assertEqual(id2.email, '*****@*****.**')
self.assertEqual(id2.username, 'jsmith')
self.assertEqual(id2.source, 'scm')
enrollments = api.enrollments(self.db, uid.uuid)
self.assertEqual(len(enrollments), 1)
def test_valid_identities_with_default_matching(self):
"""Check insertion, matching and merging of valid data"""
# First, insert the identity that will match with one
# from the file
api.add_organization(self.db, 'Example')
uuid = api.add_identity(self.db, 'unknown', email='*****@*****.**')
api.add_enrollment(self.db, uuid, 'Example',
datetime.datetime(2000, 1, 1, 0, 0),
datetime.datetime(2100, 1, 1, 0, 0))
api.edit_profile(self.db,
uuid,
name='John Smith',
is_bot=False,
country_code='US')
parser = self.get_parser(datadir('sortinghat_valid.json'))
code = self.cmd.import_identities(parser, matching='default')
self.assertEqual(code, CMD_SUCCESS)
# Check the contents of the registry
uids = api.unique_identities(self.db)
self.assertEqual(len(uids), 2)
# Jane Roe
uid = uids[0]
self.assertEqual(uid.uuid, '17ab00ed3825ec2f50483e33c88df223264182ba')
prf = uid.profile
self.assertEqual(prf.uuid, '17ab00ed3825ec2f50483e33c88df223264182ba')
self.assertEqual(prf.name, 'Jane Roe')
self.assertEqual(prf.email, '*****@*****.**')
self.assertEqual(prf.gender, None)
self.assertEqual(prf.gender_acc, None)
self.assertEqual(prf.is_bot, False)
self.assertEqual(prf.country_code, 'US')
self.assertEqual(prf.country.alpha3, 'USA')
self.assertEqual(prf.country.code, 'US')
self.assertEqual(prf.country.name, 'United States of America')
ids = self.sort_identities(uid.identities)
self.assertEqual(len(ids), 3)
id0 = ids[0]
self.assertEqual(id0.id, '17ab00ed3825ec2f50483e33c88df223264182ba')
self.assertEqual(id0.name, 'Jane Roe')
self.assertEqual(id0.email, '*****@*****.**')
self.assertEqual(id0.username, 'jroe')
self.assertEqual(id0.source, 'scm')
id1 = ids[1]
self.assertEqual(id1.id, '22d1b20763c6f5822bdda8508957486c547bb9de')
self.assertEqual(id1.name, None)
self.assertEqual(id1.email, '*****@*****.**')
self.assertEqual(id1.username, None)
self.assertEqual(id1.source, 'unknown')
id2 = ids[2]
self.assertEqual(id2.id, '322397ed782a798ffd9d0bc7e293df4292fe075d')
self.assertEqual(id2.name, None)
self.assertEqual(id2.email, '*****@*****.**')
self.assertEqual(id2.username, None)
self.assertEqual(id2.source, 'scm')
enrollments = api.enrollments(self.db, uid.uuid)
self.assertEqual(len(enrollments), 3)
# John Smith
uid = uids[1]
self.assertEqual(uid.uuid, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')
ids = self.sort_identities(uid.identities)
self.assertEqual(len(ids), 3)
# The profile was merged
prf = uid.profile
self.assertEqual(prf.uuid, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')
self.assertEqual(prf.name, 'John Smith')
self.assertEqual(prf.email, '*****@*****.**')
self.assertEqual(prf.gender, 'male')
self.assertEqual(prf.gender_acc, 100)
self.assertEqual(prf.is_bot, True)
self.assertEqual(prf.country_code, 'US')
self.assertEqual(prf.country.code, 'US')
self.assertEqual(prf.country.name, 'United States of America')
id0 = ids[0]
self.assertEqual(id0.id, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')
self.assertEqual(id0.name, None)
self.assertEqual(id0.email, '*****@*****.**')
self.assertEqual(id0.username, None)
self.assertEqual(id0.source, 'unknown')
id1 = ids[1]
self.assertEqual(id1.id, '880b3dfcb3a08712e5831bddc3dfe81fc5d7b331')
self.assertEqual(id1.name, 'John Smith')
self.assertEqual(id1.email, '*****@*****.**')
self.assertEqual(id1.username, None)
self.assertEqual(id1.source, 'scm')
id2 = ids[2]
self.assertEqual(id2.id, 'a9b403e150dd4af8953a52a4bb841051e4b705d9')
self.assertEqual(id2.name, 'John Smith')
self.assertEqual(id2.email, '*****@*****.**')
self.assertEqual(id2.username, 'jsmith')
self.assertEqual(id2.source, 'scm')
# Enrollments were merged
enrollments = api.enrollments(self.db, uid.uuid)
self.assertEqual(len(enrollments), 1)
rol0 = enrollments[0]
self.assertEqual(rol0.organization.name, 'Example')
self.assertEqual(rol0.start, datetime.datetime(2000, 1, 1, 0, 0))
self.assertEqual(rol0.end, datetime.datetime(2100, 1, 1, 0, 0))
def load_test_dataset(self):
# Add country
with self.db.connect() as session:
# Add a country
us = Country(code='US', name='United States of America', alpha3='USA')
session.add(us)
# Add identities
api.add_identity(self.db, 'scm', '*****@*****.**',
'Jane Roe', 'jroe')
jsmith_uuid = api.add_identity(self.db, 'mls', '*****@*****.**',
None, None)
api.add_identity(self.db, 'its', '*****@*****.**',
None, 'jsmith', uuid=jsmith_uuid)
api.add_identity(self.db, 'mls', '*****@*****.**',
'John Smith', 'jsmith', uuid=jsmith_uuid)
api.add_identity(self.db, 'mls', '*****@*****.**',
'J S', None, uuid=jsmith_uuid)
jdoe_uuid = api.add_identity(self.db, 'scm', '*****@*****.**',
None, 'jdoe')
api.add_identity(self.db, 'its', None,
None, 'jdoe', uuid=jdoe_uuid)
def test_valid_identities_with_default_matching(self):
"""Check insertion, matching and merging of valid data"""
# First, insert the identity that will match with one
# from the file
api.add_organization(self.db, 'Example')
uuid = api.add_identity(self.db, 'unknown', email='*****@*****.**')
api.add_enrollment(self.db, uuid, 'Example',
datetime.datetime(2000, 1, 1, 0, 0),
datetime.datetime(2100, 1, 1, 0, 0))
api.edit_profile(self.db, uuid, name='John Smith', is_bot=False,
country_code='US')
parser = self.get_parser(datadir('sortinghat_valid.json'))
code = self.cmd.import_identities(parser, matching='default')
self.assertEqual(code, CMD_SUCCESS)
# Check the contents of the registry
uids = api.unique_identities(self.db)
self.assertEqual(len(uids), 2)
# Jane Roe
uid = uids[0]
self.assertEqual(uid.uuid, '17ab00ed3825ec2f50483e33c88df223264182ba')
prf = uid.profile
self.assertEqual(prf.uuid, '17ab00ed3825ec2f50483e33c88df223264182ba')
self.assertEqual(prf.name, 'Jane Roe')
self.assertEqual(prf.email, '*****@*****.**')
self.assertEqual(prf.gender, None)
self.assertEqual(prf.gender_acc, None)
self.assertEqual(prf.is_bot, False)
self.assertEqual(prf.country_code, 'US')
self.assertEqual(prf.country.alpha3, 'USA')
self.assertEqual(prf.country.code, 'US')
self.assertEqual(prf.country.name, 'United States of America')
ids = self.sort_identities(uid.identities)
self.assertEqual(len(ids), 3)
id0 = ids[0]
self.assertEqual(id0.id, '17ab00ed3825ec2f50483e33c88df223264182ba')
self.assertEqual(id0.name, 'Jane Roe')
self.assertEqual(id0.email, '*****@*****.**')
self.assertEqual(id0.username, 'jroe')
self.assertEqual(id0.source, 'scm')
id1 = ids[1]
self.assertEqual(id1.id, '22d1b20763c6f5822bdda8508957486c547bb9de')
self.assertEqual(id1.name, None)
self.assertEqual(id1.email, '*****@*****.**')
self.assertEqual(id1.username, None)
self.assertEqual(id1.source, 'unknown')
id2 = ids[2]
self.assertEqual(id2.id, '322397ed782a798ffd9d0bc7e293df4292fe075d')
self.assertEqual(id2.name, None)
self.assertEqual(id2.email, '*****@*****.**')
self.assertEqual(id2.username, None)
self.assertEqual(id2.source, 'scm')
enrollments = api.enrollments(self.db, uid.uuid)
self.assertEqual(len(enrollments), 3)
# John Smith
uid = uids[1]
self.assertEqual(uid.uuid, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')
ids = self.sort_identities(uid.identities)
self.assertEqual(len(ids), 3)
# The profile was merged
prf = uid.profile
self.assertEqual(prf.uuid, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')
self.assertEqual(prf.name, 'John Smith')
self.assertEqual(prf.email, '*****@*****.**')
self.assertEqual(prf.gender, 'male')
self.assertEqual(prf.gender_acc, 100)
self.assertEqual(prf.is_bot, True)
self.assertEqual(prf.country_code, 'US')
self.assertEqual(prf.country.code, 'US')
self.assertEqual(prf.country.name, 'United States of America')
id0 = ids[0]
self.assertEqual(id0.id, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')
self.assertEqual(id0.name, None)
self.assertEqual(id0.email, '*****@*****.**')
self.assertEqual(id0.username, None)
self.assertEqual(id0.source, 'unknown')
id1 = ids[1]
self.assertEqual(id1.id, '880b3dfcb3a08712e5831bddc3dfe81fc5d7b331')
self.assertEqual(id1.name, 'John Smith')
self.assertEqual(id1.email, '*****@*****.**')
self.assertEqual(id1.username, None)
self.assertEqual(id1.source, 'scm')
id2 = ids[2]
self.assertEqual(id2.id, 'a9b403e150dd4af8953a52a4bb841051e4b705d9')
self.assertEqual(id2.name, 'John Smith')
self.assertEqual(id2.email, '*****@*****.**')
self.assertEqual(id2.username, 'jsmith')
self.assertEqual(id2.source, 'scm')
# Enrollments were merged
enrollments = api.enrollments(self.db, uid.uuid)
self.assertEqual(len(enrollments), 1)
rol0 = enrollments[0]
self.assertEqual(rol0.organization.name, 'Example')
self.assertEqual(rol0.start, datetime.datetime(2000, 1, 1, 0, 0))
self.assertEqual(rol0.end, datetime.datetime(2100, 1, 1, 0, 0))
def test_valid_identities_already_exist(self):
"""Check method when an identity already exists but with distinct UUID"""
# The identity already exists but with a different UUID
uuid = api.add_identity(self.db, 'unknown', email='*****@*****.**')
api.add_identity(self.db,
source='scm',
email='*****@*****.**',
name='John Smith',
username='******',
uuid=uuid)
api.edit_profile(self.db,
uuid,
name='John Smith',
is_bot=False,
country_code='US')
parser = self.get_parser('data/sortinghat_valid.json')
code = self.cmd.import_identities(parser)
self.assertEqual(code, CMD_SUCCESS)
# Check the contents of the registry
uids = api.unique_identities(self.db)
self.assertEqual(len(uids), 2)
# John Smith
uid = uids[0]
self.assertEqual(uid.uuid, '23fe3a011190a27a7c5cf6f8925de38ff0994d8d')
# The profile was not updated because it was already available
prf = uid.profile
self.assertEqual(prf.uuid, '23fe3a011190a27a7c5cf6f8925de38ff0994d8d')
self.assertEqual(prf.name, 'John Smith')
self.assertEqual(prf.email, None)
self.assertEqual(prf.is_bot, False)
self.assertEqual(prf.country_code, 'US')
self.assertEqual(prf.country.code, 'US')
self.assertEqual(prf.country.name, 'United States of America')
ids = self.sort_identities(uid.identities)
self.assertEqual(len(ids), 3)
id0 = ids[0]
self.assertEqual(id0.id, '03e12d00e37fd45593c49a5a5a1652deca4cf302')
self.assertEqual(id0.name, 'John Smith')
self.assertEqual(id0.email, '*****@*****.**')
self.assertEqual(id0.username, 'jsmith')
self.assertEqual(id0.source, 'scm')
id1 = ids[1]
self.assertEqual(id1.id, '23fe3a011190a27a7c5cf6f8925de38ff0994d8d')
self.assertEqual(id1.name, None)
self.assertEqual(id1.email, '*****@*****.**')
self.assertEqual(id1.username, None)
self.assertEqual(id1.source, 'unknown')
id2 = ids[2]
self.assertEqual(id2.id, '75d95d6c8492fd36d24a18bd45d62161e05fbc97')
self.assertEqual(id2.name, 'John Smith')
self.assertEqual(id2.email, '*****@*****.**')
self.assertEqual(id2.username, None)
self.assertEqual(id2.source, 'scm')
def test_match_new_identities(self):
"""Check whether it matches only new identities"""
parser = self.get_parser('data/sortinghat_valid.json')
code = self.cmd.import_identities(parser, matching='default')
self.assertEqual(code, CMD_SUCCESS)
# Check the contents of the registry
uids = api.unique_identities(self.db)
self.assertEqual(len(uids), 2)
# We add a new identity that matches with some of the
# already inserted
jsmith_uuid = api.add_identity(self.db,
'unknown',
email='*****@*****.**')
uids = api.unique_identities(self.db)
self.assertEqual(len(uids), 3)
# This file has a new identity, only Jane Roe will match.
parser = self.get_parser('data/sortinghat_valid_updated.json')
code = self.cmd.import_identities(parser,
matching='default',
match_new=True)
self.assertEqual(code, CMD_SUCCESS)
uids = api.unique_identities(self.db)
self.assertEqual(len(uids), 3)
# Jane Roe
uid = uids[2]
self.assertEqual(uid.uuid, '52e0aa0a14826627e633fd15332988686b730ab3')
ids = self.sort_identities(uid.identities)
self.assertEqual(len(ids), 4)
self.assertEqual(ids[0].id, '52e0aa0a14826627e633fd15332988686b730ab3')
self.assertEqual(ids[1].id, '684bb801b0ab02a8c0b6867711a994c695cbed4a')
self.assertEqual(ids[2].id, 'cbfb7bd31d556322c640f5bc7b31d58a12b15904')
self.assertEqual(ids[3].id, 'fef873c50a48cfc057f7aa19f423f81889a8907f')
# Now, if we reload again the file but setting 'match_new' to false,
# the identity that we inserted before will match "John Smith"
parser = self.get_parser('data/sortinghat_valid_updated.json')
code = self.cmd.import_identities(parser, matching='default')
self.assertEqual(code, CMD_SUCCESS)
uids = api.unique_identities(self.db)
self.assertEqual(len(uids), 2)
# John Smith
uid = uids[0]
self.assertEqual(uid.uuid, '23fe3a011190a27a7c5cf6f8925de38ff0994d8d')
ids = self.sort_identities(uid.identities)
self.assertEqual(len(ids), 3)
self.assertEqual(ids[0].id, '03e12d00e37fd45593c49a5a5a1652deca4cf302')
self.assertEqual(ids[1].id, '23fe3a011190a27a7c5cf6f8925de38ff0994d8d')
self.assertEqual(ids[1].id, jsmith_uuid)
self.assertEqual(ids[2].id, '75d95d6c8492fd36d24a18bd45d62161e05fbc97')
def test_autogender_ignore_name_not_well_formed(self):
"""Test if no gender is set when a name is invalid"""
http_requests = setup_genderize_server()
# These names are invalid so they will be ignored
uuid = api.add_identity(self.db, 'scm', '*****@*****.**',
'Random Name')
api.edit_profile(self.db, uuid, name="r4nd0m")
uuid = api.add_identity(self.db, 'mls', '*****@*****.**',
'Another Random Name')
api.edit_profile(self.db, uuid, name="ARadomName")
self.cmd.autogender(api_token='abcdefghi')
uids = api.unique_identities(self.db)
prf = uids[0].profile
self.assertEqual(prf.uuid, '2a9ec221b8dd5d5a85ae0e3276b8b2c3618ee15e')
self.assertEqual(prf.gender, 'female')
self.assertEqual(prf.gender_acc, 100)
# Jane Rae gender is not updated because it was already set
prf = uids[1].profile
self.assertEqual(prf.uuid, '3e1eccdb1e52ea56225f419d3e532fe9133c7821')
self.assertEqual(prf.gender, 'unknown')
self.assertEqual(prf.gender_acc, 100)
prf = uids[2].profile
self.assertEqual(prf.uuid, '539acca35c2e8502951a97d2d5af8b0857440b50')
self.assertEqual(prf.gender, 'male')
self.assertEqual(prf.gender_acc, 99)
prf = uids[3].profile
self.assertEqual(prf.uuid, 'a39ac334be9f17bfc7f9f21bbb25f389388f8e18')
self.assertEqual(prf.gender, 'male')
self.assertEqual(prf.gender_acc, 99)
# These names were ignored and their profile were not set either
prf = uids[4].profile
self.assertEqual(prf.uuid, 'cfa19ae04ce0c70902a31084fc75086b61ccfcf2')
self.assertEqual(prf.gender, None)
self.assertEqual(prf.gender_acc, None)
prf = uids[5].profile
self.assertEqual(prf.uuid, 'ee48da0af80479e81b846bec13fe238c06772701')
self.assertEqual(prf.gender, None)
self.assertEqual(prf.gender_acc, None)
# Check requests.
# Only two valid names were checked
expected = [
{
'name': ['jane'],
'apikey': ['abcdefghi']
},
{
'name': ['john'],
'apikey': ['abcdefghi']
},
]
self.assertEqual(len(http_requests), len(expected))
for i in range(len(expected)):
self.assertDictEqual(http_requests[i].querystring, expected[i])
def test_valid_identities_with_default_matching(self):
"""Check insertion, matching and merging of valid data"""
# First, insert the identity that will match with one
# from the file
api.add_organization(self.db, 'Example')
uuid = api.add_identity(self.db, 'unknown', email='*****@*****.**')
api.add_enrollment(self.db, uuid, 'Example',
datetime.datetime(2000, 1, 1, 0, 0),
datetime.datetime(2100, 1, 1, 0, 0))
api.edit_profile(self.db, uuid, name='John Smith', is_bot=False,
country_code='US')
parser = self.get_parser('data/sortinghat_valid.json')
code = self.cmd.import_identities(parser, matching='default')
self.assertEqual(code, CMD_SUCCESS)
# Check the contents of the registry
uids = api.unique_identities(self.db)
self.assertEqual(len(uids), 2)
# John Smith
uid = uids[0]
self.assertEqual(uid.uuid, '23fe3a011190a27a7c5cf6f8925de38ff0994d8d')
ids = self.sort_identities(uid.identities)
self.assertEqual(len(ids), 3)
# The profile was merged
prf = uid.profile
self.assertEqual(prf.uuid, '23fe3a011190a27a7c5cf6f8925de38ff0994d8d')
self.assertEqual(prf.name, 'John Smith')
self.assertEqual(prf.email, '*****@*****.**')
self.assertEqual(prf.is_bot, True)
self.assertEqual(prf.country_code, 'US')
self.assertEqual(prf.country.code, 'US')
self.assertEqual(prf.country.name, 'United States of America')
id0 = ids[0]
self.assertEqual(id0.id, '03e12d00e37fd45593c49a5a5a1652deca4cf302')
self.assertEqual(id0.name, 'John Smith')
self.assertEqual(id0.email, '*****@*****.**')
self.assertEqual(id0.username, 'jsmith')
self.assertEqual(id0.source, 'scm')
id1 = ids[1]
self.assertEqual(id1.id, '23fe3a011190a27a7c5cf6f8925de38ff0994d8d')
self.assertEqual(id1.name, None)
self.assertEqual(id1.email, '*****@*****.**')
self.assertEqual(id1.username, None)
self.assertEqual(id1.source, 'unknown')
id2 = ids[2]
self.assertEqual(id2.id, '75d95d6c8492fd36d24a18bd45d62161e05fbc97')
self.assertEqual(id2.name, 'John Smith')
self.assertEqual(id2.email, '*****@*****.**')
self.assertEqual(id2.username, None)
self.assertEqual(id2.source, 'scm')
# Enrollments were merged
enrollments = api.enrollments(self.db, uid.uuid)
self.assertEqual(len(enrollments), 1)
rol0 = enrollments[0]
self.assertEqual(rol0.organization.name, 'Example')
self.assertEqual(rol0.start, datetime.datetime(2000, 1, 1, 0, 0))
self.assertEqual(rol0.end, datetime.datetime(2100, 1, 1, 0, 0))
# Jane Roe
uid = uids[1]
self.assertEqual(uid.uuid, '52e0aa0a14826627e633fd15332988686b730ab3')
prf = uid.profile
self.assertEqual(prf.uuid, '52e0aa0a14826627e633fd15332988686b730ab3')
self.assertEqual(prf.name, 'Jane Roe')
self.assertEqual(prf.email, '*****@*****.**')
self.assertEqual(prf.is_bot, False)
self.assertEqual(prf.country_code, 'US')
self.assertEqual(prf.country.alpha3, 'USA')
self.assertEqual(prf.country.code, 'US')
self.assertEqual(prf.country.name, 'United States of America')
ids = self.sort_identities(uid.identities)
self.assertEqual(len(ids), 3)
id0 = ids[0]
self.assertEqual(id0.id, '52e0aa0a14826627e633fd15332988686b730ab3')
self.assertEqual(id0.name, 'Jane Roe')
self.assertEqual(id0.email, '*****@*****.**')
self.assertEqual(id0.username, 'jroe')
self.assertEqual(id0.source, 'scm')
id1 = ids[1]
self.assertEqual(id1.id, 'cbfb7bd31d556322c640f5bc7b31d58a12b15904')
self.assertEqual(id1.name, None)
self.assertEqual(id1.email, '*****@*****.**')
self.assertEqual(id1.username, None)
self.assertEqual(id1.source, 'unknown')
id2 = ids[2]
self.assertEqual(id2.id, 'fef873c50a48cfc057f7aa19f423f81889a8907f')
self.assertEqual(id2.name, None)
self.assertEqual(id2.email, '*****@*****.**')
self.assertEqual(id2.username, None)
self.assertEqual(id2.source, 'scm')
enrollments = api.enrollments(self.db, uid.uuid)
self.assertEqual(len(enrollments), 3)
def load_test_dataset(self):
api.add_identity(self.db, 'scm', '*****@*****.**',
'John Smith', 'jsmith')
api.add_identity(self.db, 'scm', '*****@*****.**',
'John Smith')
def add_identities(cls, db, identities, backend):
""" Load identities list from backend in Sorting Hat """
merge_identities = False
logger.info("Adding the identities to SortingHat")
if not merge_identities:
logger.info("Not doing identities merge")
total = 0
lidentities = len(identities)
if merge_identities:
merged_identities = [] # old identities merged into new ones
blacklist = api.blacklist(db)
matching = 'email-name' # Not active
matcher = create_identity_matcher(matching, blacklist)
for identity in identities:
try:
uuid = api.add_identity(db, backend, identity['email'],
identity['name'], identity['username'])
logger.debug("New sortinghat identity %s %s,%s,%s (%i/%i)" % \
(uuid, identity['username'], identity['name'], identity['email'],
total, lidentities))
total += 1
if not merge_identities:
continue # Don't do the merge here. Too slow in large projects
# Time to merge
matches = api.match_identities(db, uuid, matcher)
if len(matches) > 1:
u = api.unique_identities(db, uuid)[0]
for m in matches:
# First add the old uuid to the list of changed by merge uuids
if m.uuid not in merged_identities:
merged_identities.append(m.uuid)
if m.uuid == uuid:
continue
# Merge matched identity into added identity
api.merge_unique_identities(db, m.uuid, u.uuid)
# uuid = m.uuid
# u = api.unique_identities(db, uuid, backend)[0]
# Include all identities related to this uuid
# merged_identities.append(m.uuid)
except AlreadyExistsError as ex:
uuid = ex.uuid
continue
except WrappedValueError as ex:
logging.warning("Trying to add a None identity. Ignoring it.")
continue
except UnicodeEncodeError as ex:
logging.warning("UnicodeEncodeError. Ignoring it. %s %s %s" % \
(identity['email'], identity['name'],
identity['username']))
continue
except Exception as ex:
logging.warning("Unknown exception adding identity. Ignoring it. %s %s %s" % \
(identity['email'], identity['name'],
identity['username']))
continue
if 'company' in identity and identity['company'] is not None:
try:
api.add_organization(db, identity['company'])
api.add_enrollment(db, uuid, identity['company'],
datetime(1900, 1, 1),
datetime(2100, 1, 1))
except AlreadyExistsError:
pass
logger.info("Total NEW identities: %i" % (total))
if merge_identities:
logger.info("Total NEW identities merged: %i" % \
(len(merged_identities)))
return merged_identities
else:
return []
def test_match_new_identities(self):
"""Check whether it matches only new identities"""
parser = self.get_parser(datadir('sortinghat_valid.json'))
code = self.cmd.import_identities(parser, matching='default')
self.assertEqual(code, CMD_SUCCESS)
# Check the contents of the registry
uids = api.unique_identities(self.db)
self.assertEqual(len(uids), 2)
# We add a new identity that matches with some of the
# already inserted
jsmith_uuid = api.add_identity(self.db,
'unknown',
email='*****@*****.**')
uids = api.unique_identities(self.db)
self.assertEqual(len(uids), 3)
# This file has a new identity, only Jane Roe will match.
parser = self.get_parser(datadir('sortinghat_valid_updated.json'))
code = self.cmd.import_identities(parser,
matching='default',
match_new=True)
self.assertEqual(code, CMD_SUCCESS)
uids = api.unique_identities(self.db)
self.assertEqual(len(uids), 3)
# Jane Roe
uid = uids[0]
self.assertEqual(uid.uuid, '17ab00ed3825ec2f50483e33c88df223264182ba')
ids = self.sort_identities(uid.identities)
self.assertEqual(len(ids), 4)
self.assertEqual(ids[0].id, '17ab00ed3825ec2f50483e33c88df223264182ba')
self.assertEqual(ids[1].id, '22d1b20763c6f5822bdda8508957486c547bb9de')
self.assertEqual(ids[2].id, '322397ed782a798ffd9d0bc7e293df4292fe075d')
self.assertEqual(ids[3].id, '8ff87accaf518070bdb494b87f4d5a10e7605b47')
# Now, if we reload again the file but setting 'match_new' to false,
# the identity that we inserted before will match "John Smith"
parser = self.get_parser(datadir('sortinghat_valid_updated.json'))
code = self.cmd.import_identities(parser, matching='default')
self.assertEqual(code, CMD_SUCCESS)
uids = api.unique_identities(self.db)
self.assertEqual(len(uids), 2)
# John Smith
uid = uids[1]
self.assertEqual(uid.uuid, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')
ids = self.sort_identities(uid.identities)
self.assertEqual(len(ids), 3)
self.assertEqual(ids[0].id, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')
self.assertEqual(ids[0].id, jsmith_uuid)
self.assertEqual(ids[1].id, '880b3dfcb3a08712e5831bddc3dfe81fc5d7b331')
self.assertEqual(ids[2].id, 'a9b403e150dd4af8953a52a4bb841051e4b705d9')
def add_identities(cls, db, identities, backend):
""" Load identities list from backend in Sorting Hat """
merge_identities = False
logger.info("Adding the identities to SortingHat")
if not merge_identities:
logger.info("Not doing identities merge")
total = 0
lidentities = len(identities)
if merge_identities:
merged_identities = [] # old identities merged into new ones
blacklist = api.blacklist(db)
matching = 'email-name' # Not active
matcher = create_identity_matcher(matching, blacklist)
for identity in identities:
try:
uuid = api.add_identity(db, backend, identity['email'],
identity['name'], identity['username'])
logger.debug("New sortinghat identity %s %s,%s,%s (%i/%i)",
uuid, identity['username'], identity['name'], identity['email'],
total, lidentities)
profile = {"name": identity['name'] if identity['name'] else identity['username'],
"email": identity['email']}
api.edit_profile(db, uuid, **profile)
total += 1
if not merge_identities:
continue # Don't do the merge here. Too slow in large projects
# Time to merge
matches = api.match_identities(db, uuid, matcher)
if len(matches) > 1:
u = api.unique_identities(db, uuid)[0]
for m in matches:
# First add the old uuid to the list of changed by merge uuids
if m.uuid not in merged_identities:
merged_identities.append(m.uuid)
if m.uuid == uuid:
continue
# Merge matched identity into added identity
api.merge_unique_identities(db, m.uuid, u.uuid)
# uuid = m.uuid
# u = api.unique_identities(db, uuid, backend)[0]
# Include all identities related to this uuid
# merged_identities.append(m.uuid)
except AlreadyExistsError as ex:
uuid = ex.uuid
continue
except WrappedValueError as ex:
logging.warning("Trying to add a None identity. Ignoring it.")
continue
except UnicodeEncodeError as ex:
logging.warning("UnicodeEncodeError. Ignoring it. %s %s %s" % \
(identity['email'], identity['name'],
identity['username']))
continue
except Exception as ex:
logging.warning("Unknown exception adding identity. Ignoring it. %s %s %s" % \
(identity['email'], identity['name'],
identity['username']))
traceback.print_exc()
continue
if 'company' in identity and identity['company'] is not None:
try:
api.add_organization(db, identity['company'])
api.add_enrollment(db, uuid, identity['company'],
datetime(1900, 1, 1),
datetime(2100, 1, 1))
except AlreadyExistsError:
pass
logger.info("Total NEW identities: %i" % (total))
if merge_identities:
logger.info("Total NEW identities merged: %i" % \
(len(merged_identities)))
return merged_identities
else:
return []
def load_test_dataset(self):
# Add some unique identities
uuid = api.add_identity(self.db, source='scm', email='*****@*****.**', name='John Smith')
api.add_identity(self.db, source='scm', name='John Smith', uuid=uuid)
api.add_identity(self.db, source='scm', username='******', uuid=uuid)
uuid = api.add_identity(self.db, source='alt', name='J. Smith', username='******')
api.add_identity(self.db, source='alt', name='John Smith', username='******', uuid=uuid)
api.add_identity(self.db, source='alt', email='jsmith', uuid=uuid)
uuid = api.add_identity(self.db, source='scm', name='Jane Rae')
api.add_identity(self.db, source='mls', email='*****@*****.**', name='Jane Rae Doe', uuid=uuid)
uuid = api.add_identity(self.db, source='scm', name='J. Smith', username='******')
api.add_identity(self.db, source='scm', username='******', uuid=uuid)
api.add_identity(self.db, source='mls', username='******', uuid=uuid)
api.add_identity(self.db, source='mls', email='*****@*****.**', name='Smith. J', uuid=uuid)
uuid = api.add_identity(self.db, source='mls', email='*****@*****.**', name='Jane Rae Doe')
api.add_identity(self.db, source='mls', name='jrae', uuid=uuid)
uuid = api.add_identity(self.db, source='scm', name='jrae')
def test_reset(self):
"""Check if stored relationships and enrollments are removed before loading"""
# These identities will be split and enrollments removed
uuid = api.add_identity(self.db, 'unknown', email='*****@*****.**')
api.add_identity(self.db, source='scm', email='*****@*****.**',
name='John Smith', username='******', uuid=uuid)
api.add_organization(self.db, 'LibreSoft')
api.add_enrollment(self.db, uuid, 'LibreSoft',
datetime.datetime(2000, 1, 1, 0, 0),
datetime.datetime(2100, 1, 1, 0, 0))
parser = self.get_parser(datadir('sortinghat_valid.json'))
code = self.cmd.import_identities(parser, reset=True)
self.assertEqual(code, CMD_SUCCESS)
# Check the contents of the registry
uids = api.unique_identities(self.db)
self.assertEqual(len(uids), 3)
# Jane Roe
uid = uids[0]
self.assertEqual(uid.uuid, '17ab00ed3825ec2f50483e33c88df223264182ba')
ids = self.sort_identities(uid.identities)
self.assertEqual(len(ids), 3)
id0 = ids[0]
self.assertEqual(id0.id, '17ab00ed3825ec2f50483e33c88df223264182ba')
self.assertEqual(id0.name, 'Jane Roe')
self.assertEqual(id0.email, '*****@*****.**')
self.assertEqual(id0.username, 'jroe')
self.assertEqual(id0.source, 'scm')
enrollments = api.enrollments(self.db, uid.uuid)
self.assertEqual(len(enrollments), 3)
# [email protected]
uid = uids[1]
self.assertEqual(uid.uuid, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')
ids = self.sort_identities(uid.identities)
self.assertEqual(len(ids), 1)
id0 = ids[0]
self.assertEqual(id0.id, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')
self.assertEqual(id0.name, None)
self.assertEqual(id0.email, '*****@*****.**')
self.assertEqual(id0.username, None)
self.assertEqual(id0.source, 'unknown')
enrollments = api.enrollments(self.db, uid.uuid)
self.assertEqual(len(enrollments), 0)
# John Smith
uid = uids[2]
self.assertEqual(uid.uuid, 'a9b403e150dd4af8953a52a4bb841051e4b705d9')
ids = self.sort_identities(uid.identities)
self.assertEqual(len(ids), 2)
id1 = ids[0]
self.assertEqual(id1.id, '880b3dfcb3a08712e5831bddc3dfe81fc5d7b331')
self.assertEqual(id1.name, 'John Smith')
self.assertEqual(id1.email, '*****@*****.**')
self.assertEqual(id1.username, None)
self.assertEqual(id1.source, 'scm')
id2 = ids[1]
self.assertEqual(id2.id, 'a9b403e150dd4af8953a52a4bb841051e4b705d9')
self.assertEqual(id2.name, 'John Smith')
self.assertEqual(id2.email, '*****@*****.**')
self.assertEqual(id2.username, 'jsmith')
self.assertEqual(id2.source, 'scm')
enrollments = api.enrollments(self.db, uid.uuid)
self.assertEqual(len(enrollments), 1)
def load_test_dataset(self):
api.add_identity(self.db, 'scm', '*****@*****.**',
'John Smith', 'jsmith')
api.add_identity(self.db, 'scm', '*****@*****.**',
'John Smith')
def load_test_dataset(self):
# Add some unique identities
uuid = api.add_identity(self.db, source='scm', email='*****@*****.**', name='John Smith')
api.add_identity(self.db, source='scm', name='John Smith', uuid=uuid)
api.add_identity(self.db, source='scm', username='******', uuid=uuid)
uuid = api.add_identity(self.db, source='alt', name='J. Smith', username='******')
api.add_identity(self.db, source='alt', name='John Smith', username='******', uuid=uuid)
api.add_identity(self.db, source='alt', email='jsmith', uuid=uuid)
uuid = api.add_identity(self.db, source='scm', name='Jane Rae')
api.add_identity(self.db, source='mls', email='*****@*****.**', name='Jane Rae Doe', uuid=uuid)
uuid = api.add_identity(self.db, source='scm', name='J. Smith', username='******')
api.add_identity(self.db, source='scm', username='******', uuid=uuid)
api.add_identity(self.db, source='mls', username='******', uuid=uuid)
api.add_identity(self.db, source='mls', email='*****@*****.**', name='Smith. J', uuid=uuid)
uuid = api.add_identity(self.db, source='mls', email='*****@*****.**', name='Jane Rae Doe')
api.add_identity(self.db, source='mls', name='jrae', uuid=uuid)
uuid = api.add_identity(self.db, source='scm', name='jrae')