def test_not_valid_organizations_stream(self):
"""Check whether it parses invalid organizations files"""
# empty domains
stream_orgs = self.read_file(datadir('grimoirelab_orgs_invalid_empty_domains.yml'))
with self.assertRaisesRegex(InvalidFormatError, '^.+List of elements expected for organization Bitergia$'):
GrimoireLabParser(organizations=stream_orgs)
# one of the domains is empty
stream_orgs = self.read_file(datadir('grimoirelab_orgs_invalid_domains_list_with_empty_value.yml'))
with self.assertRaisesRegex(InvalidFormatError, '^.+Empty domain name for organization Bitergia$'):
GrimoireLabParser(organizations=stream_orgs)
# domains got a string instead of a list
stream_orgs = self.read_file(datadir('grimoirelab_orgs_invalid_wrong_domains_type.yml'))
with self.assertRaisesRegex(InvalidFormatError, '^.+List of elements expected for organization Bitergia$'):
GrimoireLabParser(organizations=stream_orgs)
# organization key missing
stream_orgs = self.read_file(datadir('grimoirelab_orgs_invalid_missing_key.yml'))
with self.assertRaisesRegex(InvalidFormatError, '^.+Attribute organization not found$'):
GrimoireLabParser(organizations=stream_orgs)
# organization key with empty value
stream_orgs = self.read_file(datadir('grimoirelab_orgs_invalid_key_with_no_value.yml'))
with self.assertRaisesRegex(InvalidFormatError, '^.+Empty organization name$'):
GrimoireLabParser(organizations=stream_orgs)
def test_email_validation(self):
aliases = self.read_file(datadir('gitdm_email_aliases_valid.txt'))
email_to_employer = self.read_file(datadir('gitdm_email_to_employer_invalid.txt'))
with self.assertRaises(InvalidFormatError):
GitdmParser(aliases=aliases,
email_to_employer=email_to_employer,
source='unknown', email_validation=True)
def test_not_valid_enrollments_parser(self):
"""Check whether data from both identites and organizations files is coherent"""
stream_ids = self.read_file(datadir('grimoirelab_invalid_datetime.yml'))
stream_orgs = self.read_file(datadir('grimoirelab_orgs_valid.yml'))
with self.assertRaises(InvalidFormatError):
GrimoireLabParser(stream_ids, stream_orgs)
def test_not_valid_stream(self):
"""Check whether it prints an error when parsing invalid streams"""
with self.assertRaisesRegex(InvalidFormatError,
MOZILLIANS_INVALID_JSON_FORMAT_ERROR):
s = self.read_file(datadir('mozillians_invalid.json'))
MozilliansParser(s)
with self.assertRaisesRegex(InvalidFormatError,
MOZILLIANS_IDS_MISSING_KEYS_ERROR):
s = self.read_file(datadir('mozillians_ids_missing_keys.json'))
MozilliansParser(s)
def test_not_valid_organizations_stream(self):
"""Check whether it raises an error when parsing invalid streams"""
with self.assertRaisesRegex(InvalidFormatError,
DOMAINS_INVALID_FORMAT_ERROR % {'line': '10'}):
stream = self.read_file(datadir('gitdm_orgs_invalid_comments.txt'))
GitdmParser(domain_to_employer=stream)
with self.assertRaisesRegex(InvalidFormatError,
DOMAINS_INVALID_FORMAT_ERROR % {'line': '8'}):
stream = self.read_file(datadir('gitdm_orgs_invalid_entries.txt'))
GitdmParser(domain_to_employer=stream)
def test_valid_organizations_stream(self):
"""Check whether it parses organizations section from a valid stream"""
stream = self.read_file(datadir('eclipse_valid.json'))
parser = EclipseParser(stream)
orgs = parser.organizations
orgs.sort(key=lambda x: x.name)
# Check parsed organizations
self.assertEqual(len(orgs), 2)
# This objects store additional information
# regarding when an organization is active or inactive
# Bitergia entry
org = orgs[0]
self.assertIsInstance(org, Organization)
self.assertEqual(org.name, 'Bitergia')
self.assertEqual(org.active, datetime.datetime(2012, 1, 1, 0, 0, 0))
self.assertEqual(org.inactive, datetime.datetime(2100, 1, 1, 0, 0, 0))
# Example entry
org = orgs[1]
self.assertIsInstance(org, Organization)
self.assertEqual(org.name, 'Example')
self.assertEqual(org.active, datetime.datetime(2009, 1, 1, 0, 0, 0))
self.assertEqual(org.inactive, datetime.datetime(2014, 9, 1, 0, 0, 0))
def test_dates_out_of_bounds(self):
"""Check dates when they are out of bounds"""
parser = self.get_parser(datadir('sortinghat_ids_dates_out_of_bounds.json'))
# This command returns a success value even when some data is wrong
code = self.cmd.import_identities(parser)
self.assertEqual(code, CMD_SUCCESS)
# Check the contents of the registry
uids = api.unique_identities(self.db)
self.assertEqual(len(uids), 1)
# Jane Roe
uid = uids[0]
self.assertEqual(uid.uuid, '17ab00ed3825ec2f50483e33c88df223264182ba')
enrollments = api.enrollments(self.db, uid.uuid)
self.assertEqual(len(enrollments), 2)
rol0 = enrollments[0]
self.assertEqual(rol0.organization.name, 'Bitergia')
self.assertEqual(rol0.start, datetime.datetime(1999, 1, 1, 0, 0))
# The json file has 2200-01-01T00:00:00
self.assertEqual(rol0.end, datetime.datetime(2100, 1, 1, 0, 0))
rol1 = enrollments[1]
self.assertEqual(rol1.organization.name, 'Example')
# The json file has 1800-01-01T00:00:00
self.assertEqual(rol1.start, datetime.datetime(1900, 1, 1, 0, 0))
self.assertEqual(rol1.end, datetime.datetime(2100, 1, 1, 0, 0))
def test_email_validation(self):
"""Check wheter it raises an error on invalid email addresses"""
stream_ids = self.read_file(datadir('grimoirelab_invalid_email.yml'))
with self.assertRaisesRegex(InvalidFormatError, '^.+Invalid email address: lcanas__at__bitergia.com$'):
GrimoireLabParser(stream_ids, email_validation=True)
def test_load_identities_invalid_file(self):
"""Test whether it prints error messages while reading invalid files"""
code = self.cmd.run('--identities', datadir('sortinghat_invalid.json'))
self.assertEqual(code, CODE_INVALID_FORMAT_ERROR)
output = sys.stderr.getvalue().strip().split('\n')[0]
self.assertEqual(output, LOAD_IDENTITIES_INVALID_JSON_FORMAT_ERROR)
def test_no_gender(self):
"""Check whether if parses identidies without gender information"""
stream = self.read_file(datadir('sortinghat_valid_no_gender.json'))
parser = SortingHatParser(stream)
uids = parser.identities
# Check parsed identities
self.assertEqual(len(uids), 1)
# John Smith
uid = uids[0]
self.assertIsInstance(uid, UniqueIdentity)
self.assertEqual(uid.uuid, '03e12d00e37fd45593c49a5a5a1652deca4cf302')
prf = uid.profile
self.assertEqual(prf.uuid, '03e12d00e37fd45593c49a5a5a1652deca4cf302')
self.assertEqual(prf.name, None)
self.assertEqual(prf.email, '*****@*****.**')
self.assertEqual(prf.gender, None)
self.assertEqual(prf.gender_acc, None)
self.assertEqual(prf.is_bot, True)
self.assertEqual(prf.country_code, None)
self.assertEqual(prf.country, None)
def test_load_organizations_overwrite(self):
"""Test to load organizations from a file with overwrite parameter set"""
code = self.cmd.run('--orgs', '--overwrite',
datadir('sortinghat_orgs_valid.json'))
self.assertEqual(code, CMD_SUCCESS)
output = sys.stdout.getvalue().strip()
self.assertEqual(output, LOAD_ORGS_OVERWRITE_OUTPUT)
def test_valid_identities_already_exist(self):
"""Check method when an identity already exists but with distinct UUID"""
# The identity already exists but with a different UUID
uuid = api.add_identity(self.db, 'unknown', email='*****@*****.**')
api.add_identity(self.db, source='scm', email='*****@*****.**',
name='John Smith', username='******', uuid=uuid)
api.edit_profile(self.db, uuid, name='John Smith', is_bot=False,
country_code='US')
parser = self.get_parser(datadir('sortinghat_valid.json'))
code = self.cmd.import_identities(parser)
self.assertEqual(code, CMD_SUCCESS)
# Check the contents of the registry
uids = api.unique_identities(self.db)
self.assertEqual(len(uids), 2)
# John Smith
uid = uids[1]
self.assertEqual(uid.uuid, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')
# The profile is updated because a new one was given
prf = uid.profile
self.assertEqual(prf.uuid, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')
self.assertEqual(prf.name, None)
self.assertEqual(prf.email, '*****@*****.**')
self.assertEqual(prf.gender, 'male')
self.assertEqual(prf.gender_acc, 100)
self.assertEqual(prf.is_bot, True)
self.assertEqual(prf.country, None)
ids = self.sort_identities(uid.identities)
self.assertEqual(len(ids), 3)
id0 = ids[0]
self.assertEqual(id0.id, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')
self.assertEqual(id0.name, None)
self.assertEqual(id0.email, '*****@*****.**')
self.assertEqual(id0.username, None)
self.assertEqual(id0.source, 'unknown')
id1 = ids[1]
self.assertEqual(id1.id, '880b3dfcb3a08712e5831bddc3dfe81fc5d7b331')
self.assertEqual(id1.name, 'John Smith')
self.assertEqual(id1.email, '*****@*****.**')
self.assertEqual(id1.username, None)
self.assertEqual(id1.source, 'scm')
id2 = ids[2]
self.assertEqual(id2.id, 'a9b403e150dd4af8953a52a4bb841051e4b705d9')
self.assertEqual(id2.name, 'John Smith')
self.assertEqual(id2.email, '*****@*****.**')
self.assertEqual(id2.username, 'jsmith')
self.assertEqual(id2.source, 'scm')
def test_invalid_matching_method(self):
"""Check if it fails when an invalid matching method is given"""
parser = self.get_parser(datadir('sortinghat_valid.json'))
code = self.cmd.import_identities(parser, matching='mock')
self.assertEqual(code, CODE_MATCHER_NOT_SUPPORTED_ERROR)
output = sys.stderr.getvalue().strip()
self.assertEqual(output, LOAD_IDENTITIES_MATCHING_ERROR)
def test_valid_organizations_file(self):
"""Check insertion of valid data from a file"""
parser = self.get_parser(datadir('sortinghat_orgs_valid.json'))
self.cmd.import_organizations(parser)
# Check the contents of the registry
orgs = api.registry(self.db)
self.assertEqual(len(orgs), 3)
# Bitergia
org = orgs[0]
self.assertEqual(org.name, 'Bitergia')
doms = org.domains
self.assertEqual(len(doms), 5)
dom = doms[0]
self.assertEqual(dom.domain, 'api.bitergia.com')
self.assertEqual(dom.is_top_domain, False)
dom = doms[1]
self.assertEqual(dom.domain, 'bitergia.com')
self.assertEqual(dom.is_top_domain, True)
dom = doms[2]
self.assertEqual(dom.domain, 'bitergia.net')
self.assertEqual(dom.is_top_domain, True)
dom = doms[3]
self.assertEqual(dom.domain, 'example.net')
self.assertEqual(dom.is_top_domain, False)
dom = doms[4]
self.assertEqual(dom.domain, 'test.bitergia.com')
self.assertEqual(dom.is_top_domain, False)
# Example
org = orgs[1]
self.assertEqual(org.name, 'Example')
doms = org.domains
self.assertEqual(len(doms), 1)
dom = doms[0]
self.assertEqual(dom.domain, 'example.com')
self.assertEqual(dom.is_top_domain, True)
# Unknown
org = orgs[2]
self.assertEqual(org.name, 'Unknown')
doms = org.domains
self.assertEqual(len(doms), 0)
def test_not_valid_stream(self):
"""Check whether it prints an error when parsing invalid streams"""
with self.assertRaisesRegex(InvalidFormatError,
SH_INVALID_JSON_FORMAT_ERROR):
s = self.read_file(datadir('sortinghat_invalid.json'))
SortingHatParser(s)
with self.assertRaisesRegex(InvalidFormatError,
SH_BL_EMPTY_STRING_ERROR):
s = self.read_file(datadir('sortinghat_blacklist_empty_strings.json'))
SortingHatParser(s)
with self.assertRaisesRegex(InvalidFormatError,
SH_IDS_MISSING_KEYS_ERROR):
s = self.read_file(datadir('sortinghat_ids_missing_keys.json'))
SortingHatParser(s)
with self.assertRaisesRegex(InvalidFormatError,
SH_ORGS_MISSING_KEYS_ERROR):
s = self.read_file(datadir('sortinghat_orgs_missing_keys.json'))
SortingHatParser(s)
with self.assertRaisesRegex(InvalidFormatError,
SH_IDS_DATETIME_ERROR):
s = self.read_file(datadir('sortinghat_ids_invalid_date.json'))
SortingHatParser(s)
with self.assertRaisesRegex(InvalidFormatError,
SH_IDS_IS_BOT_ERROR):
s = self.read_file(datadir('sortinghat_ids_invalid_is_bot.json'))
SortingHatParser(s)
with self.assertRaisesRegex(InvalidFormatError,
SH_IDS_GENDER_ACC_TYPE_ERROR):
s = self.read_file(datadir('sortinghat_ids_invalid_type_gender_acc.json'))
SortingHatParser(s)
with self.assertRaisesRegex(InvalidFormatError,
SH_IDS_GENDER_ACC_RANGE_ERROR):
s = self.read_file(datadir('sortinghat_ids_invalid_range_gender_acc.json'))
SortingHatParser(s)
with self.assertRaisesRegex(InvalidFormatError,
SH_ORGS_IS_TOP_ERROR):
s = self.read_file(datadir('sortinghat_orgs_invalid_top.json'))
SortingHatParser(s)
def test_create_profile_from_identities(self):
"""Check whether a profile is created using the data identities"""
parser = self.get_parser(datadir('sortinghat_identities_profiles.json'))
code = self.cmd.import_identities(parser)
self.assertEqual(code, CMD_SUCCESS)
# Check the contents of the registry
uids = api.unique_identities(self.db)
self.assertEqual(len(uids), 3)
# Jane Rae
uid = uids[0]
self.assertEqual(uid.uuid, '17ab00ed3825ec2f50483e33c88df223264182ba')
prf = uid.profile
self.assertEqual(prf.uuid, '17ab00ed3825ec2f50483e33c88df223264182ba')
self.assertEqual(prf.name, 'Jane Roe')
self.assertEqual(prf.email, '*****@*****.**')
self.assertEqual(prf.gender, None)
self.assertEqual(prf.gender_acc, None)
self.assertEqual(prf.is_bot, False)
self.assertEqual(prf.country_code, None)
self.assertEqual(prf.country, None)
# John Smith
uid = uids[1]
self.assertEqual(uid.uuid, 'a9b403e150dd4af8953a52a4bb841051e4b705d9')
prf = uid.profile
self.assertEqual(prf.uuid, 'a9b403e150dd4af8953a52a4bb841051e4b705d9')
self.assertEqual(prf.name, 'John Smith')
self.assertEqual(prf.email, '*****@*****.**')
self.assertEqual(prf.gender, None)
self.assertEqual(prf.gender_acc, None)
self.assertEqual(prf.is_bot, False)
self.assertEqual(prf.country_code, None)
self.assertEqual(prf.country, None)
# John Doe
uid = uids[2]
self.assertEqual(uid.uuid, 'c2f8c3d7b49cdbfb0af9fc9db2ca098ec6c06c2f')
prf = uid.profile
self.assertEqual(prf.uuid, 'c2f8c3d7b49cdbfb0af9fc9db2ca098ec6c06c2f')
self.assertEqual(prf.name, 'jdoe')
self.assertEqual(prf.email, '*****@*****.**')
self.assertEqual(prf.gender, None)
self.assertEqual(prf.gender_acc, None)
self.assertEqual(prf.is_bot, False)
self.assertEqual(prf.country_code, None)
self.assertEqual(prf.country, None)
def test_invalid_format(self):
"""Check whether it prints an error when parsing invalid files"""
code = self.cmd.run(datadir('sortinghat_invalid.json'))
self.assertEqual(code, CODE_INVALID_FORMAT_ERROR)
output = sys.stderr.getvalue().strip().split('\n')[0]
self.assertEqual(output, LOAD_IDENTITIES_INVALID_JSON_FORMAT_ERROR)
code = self.cmd.run(datadir('sortinghat_ids_missing_keys.json'))
self.assertEqual(code, CODE_INVALID_FORMAT_ERROR)
output = sys.stderr.getvalue().strip().split('\n')[1]
self.assertEqual(output, LOAD_IDENTITIES_MISSING_KEYS_ERROR)
# Context added to catch deprecation warnings raised on Python 3
with warnings.catch_warnings(record=True):
code = self.cmd.run(datadir('sortinghat_orgs_invalid_json.json'))
self.assertEqual(code, CODE_INVALID_FORMAT_ERROR)
output = sys.stderr.getvalue().strip().split('\n')[2]
self.assertRegexpMatches(output, LOAD_ORGS_INVALID_FORMAT_ERROR)
code = self.cmd.run(datadir('sortinghat_orgs_missing_keys.json'))
self.assertEqual(code, CODE_INVALID_FORMAT_ERROR)
output = sys.stderr.getvalue().strip().split('\n')[3]
self.assertEqual(output, LOAD_ORGS_MISSING_KEYS_ERROR)
code = self.cmd.run(datadir('sortinghat_orgs_invalid_top.json'))
self.assertEqual(code, CODE_INVALID_FORMAT_ERROR)
output = sys.stderr.getvalue().strip().split('\n')[4]
self.assertEqual(output, LOAD_ORGS_IS_TOP_ERROR)
code = self.cmd.run(datadir('sortinghat_blacklist_empty_strings.json'))
self.assertEqual(code, CODE_INVALID_FORMAT_ERROR)
output = sys.stderr.getvalue().strip().split('\n')[5]
self.assertEqual(output, LOAD_BLACKLIST_EMPTY_STRINGS_ERROR)
def test_not_valid_identities_stream(self):
"""Check whether it parses invalid identities files"""
stream_ids = self.read_file(datadir('grimoirelab_invalid_email.yml'))
with self.assertRaisesRegex(InvalidFormatError, '^.+Invalid email address: lcanas__at__bitergia.com$'):
GrimoireLabParser(stream_ids)
stream_ids = self.read_file(datadir('grimoirelab_invalid_structure.yml'))
with self.assertRaisesRegex(InvalidFormatError, '^.+Attribute profile not found$'):
GrimoireLabParser(stream_ids)
stream_ids = self.read_file(datadir('grimoirelab_invalid_missing_accounts.yml'))
with self.assertRaisesRegex(InvalidFormatError, '^.+Attribute name not found$'):
GrimoireLabParser(stream_ids)
stream_ids = self.read_file(datadir('grimoirelab_invalid_missing_profile_name_isbot.yml'))
with self.assertRaisesRegex(InvalidFormatError, '^.+Attribute name not found$'):
GrimoireLabParser(stream_ids)
stream_ids = self.read_file(datadir('grimoirelab_invalid_missing_profile.yml'))
with self.assertRaisesRegex(InvalidFormatError, '^.+Attribute profile not found$'):
GrimoireLabParser(stream_ids)
stream_ids = self.read_file(datadir('grimoirelab_invalid_missing_organization_name.yml'))
with self.assertRaisesRegex(InvalidFormatError, '^.+Empty organization name$'):
GrimoireLabParser(stream_ids)
def test_not_valid_stream(self):
"""Check whether it prints an error when parsing invalid streams"""
with self.assertRaisesRegex(InvalidFormatError,
ECLIPSE_INVALID_JSON_FORMAT_ERROR):
s = self.read_file(datadir('eclipse_invalid.json'))
EclipseParser(s)
with self.assertRaisesRegex(InvalidFormatError,
ECLIPSE_IDS_MISSING_KEYS_ERROR):
s = self.read_file(datadir('eclipse_ids_missing_keys.json'))
EclipseParser(s)
with self.assertRaisesRegex(InvalidFormatError,
ECLIPSE_ORGS_MISSING_KEYS_ERROR):
s = self.read_file(datadir('eclipse_orgs_missing_keys.json'))
EclipseParser(s)
with self.assertRaisesRegex(InvalidFormatError,
ECLIPSE_DATETIME_ERROR):
s = self.read_file(datadir('eclipse_invalid_datetime.json'))
EclipseParser(s)
def test_invalid_config_files(self):
"""Check whether it raises and error reading invalid configuration files"""
# Test directory
dirpath = os.path.expanduser('~')
self.assertRaises(RuntimeError, self.cmd.get,
'db.user', dirpath)
# Test non existing file
self.assertRaisesRegex(RuntimeError, NOT_FOUND_FILE_ERROR,
self.cmd.get, 'db.user',
datadir('invalid_config_file.cfg'))
def test_load_identities_with_default_matching(self):
"""Test to load identities from a file using default matching"""
code = self.cmd.run('--identities', '--matching', 'default',
datadir('sortinghat_valid.json'),
'--verbose')
self.assertEqual(code, CMD_SUCCESS)
output = sys.stdout.getvalue().strip()
self.assertEqual(output, LOAD_IDENTITIES_OUTPUT)
output = sys.stderr.getvalue().strip()
self.assertEqual(output, LOAD_IDENTITIES_OUTPUT_ERROR)
def test_load(self):
"""Test to load identities and organizations from a file"""
code = self.cmd.run(datadir('sortinghat_valid.json'), '--verbose')
self.assertEqual(code, CMD_SUCCESS)
uids = api.unique_identities(self.db)
self.assertEqual(len(uids), 2)
orgs = api.registry(self.db)
self.assertEqual(len(orgs), 3)
output = sys.stdout.getvalue().strip()
self.assertEqual(output, LOAD_OUTPUT)
def test_valid_organizations_stream(self):
"""Check whether it parses organizations section from a valid stream"""
stream = self.read_file(datadir('mozillians_valid.json'))
parser = MozilliansParser(stream)
orgs = parser.organizations
# Check parsed organizations
self.assertEqual(len(orgs), 1)
org = orgs[0]
self.assertIsInstance(org, Organization)
self.assertEqual(org.name, MOZILLIANS_ORG)
def test_load_identities_no_strict_matching(self):
"""Test to load identities with no strict matching"""
# First, insert the identity that will match with one
# from the file
api.add_identity(self.db, 'unknown', email='jsmith@example')
code = self.cmd.run('--identities', '--matching', 'default',
'--no-strict-matching',
datadir('sortinghat_no_strict_valid.json'),
'--verbose')
self.assertEqual(code, CMD_SUCCESS)
output = sys.stdout.getvalue().strip()
self.assertEqual(output, LOAD_IDENTITIES_NO_STRICT_OUTPUT)
def test_load_organizations(self):
"""Test to load organizations from a file"""
code = self.cmd.run('--orgs', datadir('sortinghat_orgs_valid.json'))
self.assertEqual(code, CMD_SUCCESS)
uids = api.unique_identities(self.db)
self.assertEqual(len(uids), 0)
orgs = api.registry(self.db)
self.assertEqual(len(orgs), 3)
output = sys.stdout.getvalue().strip()
self.assertEqual(output, LOAD_SH_ORGS_OUTPUT)
output = sys.stderr.getvalue().strip()
self.assertEqual(output, LOAD_ORGS_OUTPUT_WARNING)
def test_valid_file(self):
"""Check insertion of valid data from a file"""
parser = self.get_parser(datadir('sortinghat_valid.json'))
self.cmd.import_blacklist(parser)
# Check the contents of the registry
bl = api.blacklist(self.db)
self.assertEqual(len(bl), 2)
# John Smith
b = bl[0]
self.assertEqual(b.excluded, 'John Smith')
b = bl[1]
self.assertEqual(b.excluded, '*****@*****.**')
def test_matching_no_strict(self):
"""Check if identities with no strict matching are merged"""
# First, insert the identity that will match with one
# from the file
api.add_identity(self.db, 'unknown', email='jsmith@example')
parser = self.get_parser(datadir('sortinghat_no_strict_valid.json'))
code = self.cmd.import_identities(parser, matching='default',
no_strict_matching=True)
self.assertEqual(code, CMD_SUCCESS)
# Check whether identities were merged
uids = api.unique_identities(self.db)
self.assertEqual(len(uids), 1)
self.assertEqual(len(uids[0].identities), 2)
def test_valid_blacklist_stream(self):
"""Check whether it parsers blacklist section from a valid stream"""
stream = self.read_file(datadir('sortinghat_valid.json'))
parser = SortingHatParser(stream)
bl = parser.blacklist
# Check parsed blacklist
self.assertEqual(len(bl), 2)
b = bl[0]
self.assertIsInstance(b, MatchingBlacklist)
self.assertEqual(b.excluded, 'John Smith')
b = bl[1]
self.assertIsInstance(b, MatchingBlacklist)
self.assertEqual(b.excluded, '*****@*****.**')
def test_load_identities(self):
"""Test to load identities from a file"""
code = self.cmd.run('--identities', datadir('sortinghat_valid.json'),
'--verbose')
self.assertEqual(code, CMD_SUCCESS)
uids = api.unique_identities(self.db)
self.assertEqual(len(uids), 2)
# This has imported the organizations from the enrollments,
# not thouse from organizations section
orgs = api.registry(self.db)
self.assertEqual(len(orgs), 2)
output = sys.stdout.getvalue().strip()
self.assertEqual(output, LOAD_IDENTITIES_OUTPUT)
output = sys.stderr.getvalue().strip()
self.assertEqual(output, LOAD_IDENTITIES_OUTPUT_ERROR)
def test_enrollments_parser(self):
"""Check whether enrollments are correctly parsed"""
stream_ids = self.read_file(datadir('grimoirelab_valid.yml'))
parser = GrimoireLabParser(stream_ids)
# Parsed unique identities
uids = parser.identities
# J. Manrique Lopez de la Fuente
uid = uids[0]
rol = uid.enrollments[0]
self.assertIsInstance(rol, Enrollment)
self.assertEqual(rol.organization.name, 'Bitergia')
self.assertEqual(rol.start, datetime.datetime(2013, 1, 1, 0, 0))
self.assertEqual(rol.end, datetime.datetime(2100, 1, 1, 0, 0))
# Luis Cañas-Díaz
uid = uids[1]
# Unknown organization is ignored during the parsing process
self.assertEqual(len(uid.enrollments), 2)
rol = uid.enrollments[0]
self.assertIsInstance(rol, Enrollment)
self.assertEqual(rol.organization.name, 'GSyC/LibreSoft')
self.assertEqual(rol.start, datetime.datetime(2003, 1, 1, 0, 0))
self.assertEqual(rol.end, datetime.datetime(2011, 12, 31, 0, 0))
rol = uid.enrollments[1]
self.assertIsInstance(rol, Enrollment)
self.assertEqual(rol.organization.name, 'Bitergia')
self.assertEqual(rol.start, datetime.datetime(2012, 1, 1, 0, 0))
self.assertEqual(rol.end, datetime.datetime(2100, 1, 1, 0, 0))
# Owl Bot
uid = uids[2]
# Unknown organization is ignored during the parsing process
self.assertEqual(len(uid.enrollments), 0)
def test_valid_identities_stream(self):
"""Check whether it parsers identities section from a valid stream"""
stream = self.read_file(datadir('sortinghat_valid.json'))
parser = SortingHatParser(stream)
uids = parser.identities
# Check parsed identities
self.assertEqual(len(uids), 3)
# 00000 identity
uid = uids[0]
self.assertIsInstance(uid, UniqueIdentity)
self.assertEqual(uid.uuid, '0000000000000000000000000000000000000000')
prf = uid.profile
self.assertEqual(prf.uuid, '0000000000000000000000000000000000000000')
self.assertEqual(prf.name, None)
self.assertEqual(prf.email, None)
self.assertEqual(prf.gender, None)
self.assertEqual(prf.gender_acc, None)
self.assertEqual(prf.is_bot, False)
self.assertEqual(prf.country_code, None)
self.assertEqual(prf.country, None)
self.assertEqual(len(uid.identities), 0)
self.assertEqual(len(uid.enrollments), 0)
# John Smith
uid = uids[1]
self.assertIsInstance(uid, UniqueIdentity)
self.assertEqual(uid.uuid, '03e12d00e37fd45593c49a5a5a1652deca4cf302')
prf = uid.profile
self.assertEqual(prf.uuid, '03e12d00e37fd45593c49a5a5a1652deca4cf302')
self.assertEqual(prf.name, None)
self.assertEqual(prf.email, '*****@*****.**')
self.assertEqual(prf.gender, 'male')
self.assertEqual(prf.gender_acc, 100)
self.assertEqual(prf.is_bot, True)
self.assertEqual(prf.country_code, None)
self.assertEqual(prf.country, None)
ids = uid.identities
self.assertEqual(len(ids), 2)
id0 = ids[0]
self.assertEqual(id0.id, '03e12d00e37fd45593c49a5a5a1652deca4cf302')
self.assertEqual(id0.name, 'John Smith')
self.assertEqual(id0.email, '*****@*****.**')
self.assertEqual(id0.username, 'jsmith')
self.assertEqual(id0.source, 'scm')
self.assertEqual(id0.uuid, '03e12d00e37fd45593c49a5a5a1652deca4cf302')
id1 = ids[1]
self.assertEqual(id1.id, '75d95d6c8492fd36d24a18bd45d62161e05fbc97')
self.assertEqual(id1.name, 'John Smith')
self.assertEqual(id1.email, '*****@*****.**')
self.assertEqual(id1.username, None)
self.assertEqual(id1.source, 'scm')
self.assertEqual(id1.uuid, '03e12d00e37fd45593c49a5a5a1652deca4cf302')
enrollments = uid.enrollments
self.assertEqual(len(enrollments), 1)
rol0 = enrollments[0]
self.assertIsInstance(rol0.organization, Organization)
self.assertEqual(rol0.organization.name, 'Example')
self.assertEqual(rol0.start, datetime.datetime(1900, 1, 1, 0, 0))
self.assertEqual(rol0.end, datetime.datetime(2100, 1, 1, 0, 0))
# Jane Roe
uid = uids[2]
self.assertIsInstance(uid, UniqueIdentity)
self.assertEqual(uid.uuid, '52e0aa0a14826627e633fd15332988686b730ab3')
prf = uid.profile
self.assertEqual(prf.uuid, '52e0aa0a14826627e633fd15332988686b730ab3')
self.assertEqual(prf.name, 'Jane Roe')
self.assertEqual(prf.email, '*****@*****.**')
self.assertEqual(prf.gender, None)
self.assertEqual(prf.gender_acc, None)
self.assertEqual(prf.is_bot, False)
self.assertEqual(prf.country_code, 'US')
self.assertEqual(prf.country.alpha3, 'USA')
self.assertEqual(prf.country.code, 'US')
self.assertEqual(prf.country.name, 'United States of America')
ids = uid.identities
self.assertEqual(len(ids), 3)
id0 = ids[0]
self.assertEqual(id0.id, '52e0aa0a14826627e633fd15332988686b730ab3')
self.assertEqual(id0.name, 'Jane Roe')
self.assertEqual(id0.email, '*****@*****.**')
self.assertEqual(id0.username, 'jroe')
self.assertEqual(id0.source, 'scm')
self.assertEqual(id0.uuid, '52e0aa0a14826627e633fd15332988686b730ab3')
id1 = ids[1]
self.assertEqual(id1.id, 'cbfb7bd31d556322c640f5bc7b31d58a12b15904')
self.assertEqual(id1.name, None)
self.assertEqual(id1.email, '*****@*****.**')
self.assertEqual(id1.username, None)
self.assertEqual(id1.source, 'unknown')
self.assertEqual(id0.uuid, '52e0aa0a14826627e633fd15332988686b730ab3')
id2 = ids[2]
self.assertEqual(id2.id, 'fef873c50a48cfc057f7aa19f423f81889a8907f')
self.assertEqual(id2.name, None)
self.assertEqual(id2.email, '*****@*****.**')
self.assertEqual(id2.username, None)
self.assertEqual(id2.source, 'scm')
self.assertEqual(id0.uuid, '52e0aa0a14826627e633fd15332988686b730ab3')
enrollments = uid.enrollments
self.assertEqual(len(enrollments), 3)
rol0 = enrollments[0]
self.assertIsInstance(rol0.organization, Organization)
self.assertEqual(rol0.organization.name, 'Bitergia')
self.assertEqual(rol0.start, datetime.datetime(1999, 1, 1, 0, 0))
self.assertEqual(rol0.end, datetime.datetime(2000, 1, 1, 0, 0))
rol1 = enrollments[1]
self.assertIsInstance(rol0.organization, Organization)
self.assertEqual(rol1.organization.name, 'Bitergia')
self.assertEqual(rol1.start, datetime.datetime(2006, 1, 1, 0, 0))
self.assertEqual(rol1.end, datetime.datetime(2008, 1, 1, 0, 0))
rol2 = enrollments[2]
self.assertIsInstance(rol0.organization, Organization)
self.assertEqual(rol2.organization.name, 'Example')
self.assertEqual(rol2.start, datetime.datetime(1900, 1, 1, 0, 0))
self.assertEqual(rol2.end, datetime.datetime(2100, 1, 1, 0, 0))
def test_valid_organizations_stream(self):
"""Check whether it parses organizations section from a valid stream"""
stream = self.read_file(datadir('sortinghat_valid.json'))
parser = SortingHatParser(stream)
orgs = parser.organizations
# Check parsed organizations
self.assertEqual(len(orgs), 3)
# Bitergia entry
org = orgs[0]
self.assertIsInstance(org, Organization)
self.assertEqual(org.name, 'Bitergia')
doms = org.domains
self.assertEqual(len(doms), 4)
dom = doms[0]
self.assertIsInstance(dom, Domain)
self.assertEqual(dom.domain, 'api.bitergia.com')
self.assertEqual(dom.is_top_domain, False)
dom = doms[1]
self.assertIsInstance(dom, Domain)
self.assertEqual(dom.domain, 'bitergia.com')
self.assertEqual(dom.is_top_domain, True)
dom = doms[2]
self.assertIsInstance(dom, Domain)
self.assertEqual(dom.domain, 'bitergia.net')
self.assertEqual(dom.is_top_domain, True)
dom = doms[3]
self.assertIsInstance(dom, Domain)
self.assertEqual(dom.domain, 'test.bitergia.com')
self.assertEqual(dom.is_top_domain, False)
# Example entry
org = orgs[1]
self.assertIsInstance(org, Organization)
self.assertEqual(org.name, 'Example')
doms = org.domains
self.assertEqual(len(doms), 2)
dom = doms[0]
self.assertIsInstance(dom, Domain)
self.assertEqual(dom.domain, 'example.com')
self.assertEqual(dom.is_top_domain, True)
dom = doms[1]
self.assertIsInstance(dom, Domain)
self.assertEqual(dom.domain, 'example.net')
self.assertEqual(dom.is_top_domain, True)
# Unknown entry
org = orgs[2]
self.assertIsInstance(org, Organization)
self.assertEqual(org.name, 'Unknown')
doms = org.domains
self.assertEqual(len(doms), 0)
def test_organizations_parser(self):
"""Check whether it parses a valid organizations file"""
stream_orgs = self.read_file(datadir('grimoirelab_orgs_valid.yml'))
parser = GrimoireLabParser(organizations=stream_orgs)
# Parsed organizations
orgs = parser.organizations
self.assertEqual(len(orgs), 3)
# Bitergia entries
org = orgs[0]
self.assertIsInstance(org, Organization)
self.assertEqual(org.name, 'Bitergia')
doms = org.domains
self.assertEqual(len(doms), 2)
dom = doms[0]
self.assertIsInstance(dom, Domain)
self.assertEqual(dom.domain, 'bitergia.com')
self.assertEqual(dom.is_top_domain, False)
dom = doms[1]
self.assertIsInstance(dom, Domain)
self.assertEqual(dom.domain, 'bitergia.net')
self.assertEqual(dom.is_top_domain, False)
# Example entries
org = orgs[1]
self.assertIsInstance(org, Organization)
self.assertEqual(org.name, 'Example')
doms = org.domains
self.assertEqual(len(doms), 3)
dom = doms[0]
self.assertIsInstance(dom, Domain)
self.assertEqual(dom.domain, 'example.com')
self.assertEqual(dom.is_top_domain, False)
dom = doms[1]
self.assertIsInstance(dom, Domain)
self.assertEqual(dom.domain, 'example.org')
self.assertEqual(dom.is_top_domain, False)
dom = doms[2]
self.assertIsInstance(dom, Domain)
self.assertEqual(dom.domain, 'example.net')
self.assertEqual(dom.is_top_domain, False)
# GSyC/Libresof entries
org = orgs[2]
self.assertIsInstance(org, Organization)
self.assertEqual(org.name, 'GSyC/LibreSoft')
doms = org.domains
self.assertEqual(len(doms), 2)
dom = doms[0]
self.assertIsInstance(dom, Domain)
self.assertEqual(dom.domain, 'libresoft.es')
self.assertEqual(dom.is_top_domain, False)
dom = doms[1]
self.assertIsInstance(dom, Domain)
self.assertEqual(dom.domain, 'gsyc.es')
self.assertEqual(dom.is_top_domain, False)
def test_identities_parser(self):
"""Check whether it parses a valid identities file"""
stream_ids = self.read_file(datadir('grimoirelab_valid.yml'))
parser = GrimoireLabParser(stream_ids)
# Parsed unique identities
uids = parser.identities
self.assertEqual(len(uids), 3)
# J. Manrique Lopez de la Fuente
uid = uids[0]
self.assertIsInstance(uid, UniqueIdentity)
self.assertFalse(uid.profile.is_bot)
self.assertEqual(uid.profile.name, 'J. Manrique Lopez de la Fuente')
ids = uid.identities
self.assertEqual(len(ids), 4)
id0 = ids[0]
self.assertIsInstance(id0, Identity)
self.assertEqual(id0.name, 'J. Manrique Lopez de la Fuente')
self.assertEqual(id0.email, None)
self.assertEqual(id0.username, None)
self.assertEqual(id0.source, 'grimoirelab')
self.assertEqual(id0.uuid, None)
id1 = ids[1]
self.assertIsInstance(id1, Identity)
self.assertEqual(id1.name, None)
self.assertEqual(id1.email, '*****@*****.**')
self.assertEqual(id1.username, None)
self.assertEqual(id1.source, 'grimoirelab')
self.assertEqual(id1.uuid, None)
id1 = ids[2]
self.assertIsInstance(id1, Identity)
self.assertEqual(id1.name, None)
self.assertEqual(id1.email, None)
self.assertEqual(id1.username, 'jsmanrique')
self.assertEqual(id1.source, 'github')
self.assertEqual(id1.uuid, None)
id1 = ids[3]
self.assertIsInstance(id1, Identity)
self.assertEqual(id1.name, None)
self.assertEqual(id1.email, None)
self.assertEqual(id1.username, 'jsm')
self.assertEqual(id1.source, 'jira')
self.assertEqual(id1.uuid, None)
# Luis Cañas-Díaz
uid = uids[1]
self.assertIsInstance(uid, UniqueIdentity)
self.assertFalse(uid.profile.is_bot)
self.assertEqual(uid.profile.name, 'Luis Cañas-Díaz')
self.assertIsInstance(uid, UniqueIdentity)
ids = uid.identities
self.assertEqual(len(ids), 3)
id0 = ids[0]
self.assertIsInstance(id0, Identity)
self.assertEqual(id0.name, 'Luis Cañas-Díaz')
self.assertEqual(id0.email, None)
self.assertEqual(id0.username, None)
self.assertEqual(id0.source, 'grimoirelab')
self.assertEqual(id0.uuid, None)
id1 = ids[1]
self.assertIsInstance(id1, Identity)
self.assertEqual(id1.name, None)
self.assertEqual(id1.email, '*****@*****.**')
self.assertEqual(id1.username, None)
self.assertEqual(id1.source, 'grimoirelab')
self.assertEqual(id1.uuid, None)
id1 = ids[2]
self.assertIsInstance(id1, Identity)
self.assertEqual(id1.name, None)
self.assertEqual(id1.email, None)
self.assertEqual(id1.username, 'sanacl')
self.assertEqual(id1.source, 'github')
self.assertEqual(id1.uuid, None)
# owlbot
uid = uids[2]
self.assertIsInstance(uid, UniqueIdentity)
self.assertTrue(uid.profile.is_bot)
self.assertIsInstance(uid, UniqueIdentity)
ids = uid.identities
self.assertEqual(len(ids), 2)
id0 = ids[0]
self.assertIsInstance(id0, Identity)
self.assertEqual(id0.name, 'Owl Bot')
self.assertEqual(id0.email, None)
self.assertEqual(id0.username, None)
self.assertEqual(id0.source, 'grimoirelab')
self.assertEqual(id0.uuid, None)
id0 = ids[1]
self.assertIsInstance(id0, Identity)
self.assertEqual(id0.name, None)
self.assertEqual(id0.email, '*****@*****.**')
self.assertEqual(id0.username, None)
self.assertEqual(id0.source, 'grimoirelab')
self.assertEqual(id0.uuid, None)
def test_valid_identities_stream(self):
"""Check insertion of valid data from a file"""
stream = self.read_file(datadir('mozillians_valid.json'))
parser = MozilliansParser(stream, source='unknown')
uids = parser.identities
# Check parsed identities
self.assertEqual(len(uids), 3)
# John Doe identity
uid = uids[0]
self.assertIsInstance(uid, UniqueIdentity)
self.assertEqual(uid.uuid, 'john_doe')
self.assertEqual(len(uid.identities), 2)
id0 = uid.identities[0]
self.assertIsInstance(id0, Identity)
self.assertEqual(id0.id, None)
self.assertEqual(id0.name, None)
self.assertEqual(id0.email, None)
self.assertEqual(id0.username, 'john_doe')
self.assertEqual(id0.uuid, 'john_doe')
self.assertEqual(id0.source, 'unknown')
# IRC identity
id1 = uid.identities[1]
self.assertIsInstance(id1, Identity)
self.assertEqual(id1.id, None)
self.assertEqual(id1.name, None)
self.assertEqual(id1.email, None)
self.assertEqual(id1.username, 'jdoe')
self.assertEqual(id1.uuid, 'john_doe')
self.assertEqual(id1.source, 'unknown')
enrollments = uid.enrollments
self.assertEqual(len(enrollments), 1)
rol0 = enrollments[0]
self.assertIsInstance(rol0.organization, Organization)
self.assertEqual(rol0.organization.name, MOZILLIANS_ORG)
self.assertEqual(rol0.start, datetime.datetime(2015, 6, 1, 0, 0, 0))
# Jane Rae unique identity
uid = uids[1]
self.assertIsInstance(uid, UniqueIdentity)
self.assertEqual(uid.uuid, 'jrae')
self.assertEqual(len(uid.identities), 3)
id0 = uid.identities[0]
self.assertIsInstance(id0, Identity)
self.assertEqual(id0.id, None)
self.assertEqual(id0.name, 'Jane Rae')
self.assertEqual(id0.email, None)
self.assertEqual(id0.username, 'jrae')
self.assertEqual(id0.uuid, 'jrae')
self.assertEqual(id0.source, 'unknown')
id1 = uid.identities[1]
self.assertIsInstance(id1, Identity)
self.assertEqual(id1.id, None)
self.assertEqual(id1.name, 'Jane Rae')
self.assertEqual(id1.email, '*****@*****.**')
self.assertEqual(id1.username, 'jrae')
self.assertEqual(id1.uuid, 'jrae')
self.assertEqual(id1.source, 'unknown')
id2 = uid.identities[2]
self.assertIsInstance(id2, Identity)
self.assertEqual(id2.id, None)
self.assertEqual(id2.name, 'Jane Rae')
self.assertEqual(id2.email, '*****@*****.**')
self.assertEqual(id2.username, 'jrae')
self.assertEqual(id2.uuid, 'jrae')
self.assertEqual(id2.source, 'unknown')
enrollments = uid.enrollments
self.assertEqual(len(enrollments), 1)
rol0 = enrollments[0]
self.assertIsInstance(rol0.organization, Organization)
self.assertEqual(rol0.organization.name, MOZILLIANS_ORG)
self.assertEqual(rol0.start, datetime.datetime(1900, 1, 1, 0, 0, 0))
# John Smith unique identity
uid = uids[2]
self.assertIsInstance(uid, UniqueIdentity)
self.assertEqual(uid.uuid, 'jsmith')
self.assertEqual(len(uid.identities), 1)
# Only one identity found. No alternative emails
# and ircname is equal to username value
id0 = uid.identities[0]
self.assertIsInstance(id0, Identity)
self.assertEqual(id0.id, None)
self.assertEqual(id0.name, 'John Smith')
self.assertEqual(id0.email, '*****@*****.**')
self.assertEqual(id0.username, 'jsmith')
self.assertEqual(id0.uuid, 'jsmith')
self.assertEqual(id0.source, 'unknown')
enrollments = uid.enrollments
self.assertEqual(len(enrollments), 1)
rol0 = enrollments[0]
self.assertIsInstance(rol0.organization, Organization)
self.assertEqual(rol0.organization.name, MOZILLIANS_ORG)
self.assertEqual(rol0.start, datetime.datetime(1900, 1, 1, 0, 0, 0))
def test_valid_identities_stream(self):
"""Check insertion of valid data from a file"""
stream = self.read_file(datadir('eclipse_valid.json'))
parser = EclipseParser(stream, source='unknown')
uids = parser.identities
# Check parsed identities
self.assertEqual(len(uids), 3)
# John Doe unique identity
uid = uids[0]
self.assertIsInstance(uid, UniqueIdentity)
self.assertEqual(uid.uuid, 'jdoe')
self.assertEqual(len(uid.identities), 1)
id0 = uid.identities[0]
self.assertIsInstance(id0, Identity)
self.assertEqual(id0.id, None)
self.assertEqual(id0.name, 'John Doe')
self.assertEqual(id0.email, '*****@*****.**')
self.assertEqual(id0.username, 'jdoe')
self.assertEqual(id0.uuid, 'jdoe')
self.assertEqual(id0.source, 'unknown')
enrollments = uid.enrollments
self.assertEqual(len(enrollments), 1)
rol0 = enrollments[0]
self.assertIsInstance(rol0.organization, Organization)
self.assertEqual(rol0.organization.name, 'Example')
self.assertEqual(rol0.start, datetime.datetime(2010, 1, 1, 0, 0, 0))
# The parsed end date shoud be 2100-01-01 but because there is
# an inactive date for Example, this end date is set to the
# inactive date
self.assertEqual(rol0.end, datetime.datetime(2014, 9, 1, 0, 0, 0))
# Jane Rae identity
uid = uids[1]
self.assertIsInstance(uid, UniqueIdentity)
self.assertEqual(uid.uuid, 'jrae')
self.assertEqual(len(uid.identities), 1)
id0 = uid.identities[0]
self.assertIsInstance(id0, Identity)
self.assertEqual(id0.id, None)
self.assertEqual(id0.name, 'Jane Rae')
self.assertEqual(id0.email, '*****@*****.**')
self.assertEqual(id0.username, 'jrae')
self.assertEqual(id0.uuid, 'jrae')
self.assertEqual(id0.source, 'unknown')
enrollments = uid.enrollments
self.assertEqual(len(enrollments), 0)
# John Smith unique identity
uid = uids[2]
self.assertIsInstance(uid, UniqueIdentity)
self.assertEqual(uid.uuid, 'jsmith')
self.assertEqual(len(uid.identities), 2)
id0 = uid.identities[0]
self.assertIsInstance(id0, Identity)
self.assertEqual(id0.id, None)
self.assertEqual(id0.name, 'John Smith')
self.assertEqual(id0.email, '*****@*****.**')
self.assertEqual(id0.username, 'jsmith')
self.assertEqual(id0.uuid, 'jsmith')
self.assertEqual(id0.source, 'unknown')
id1 = uid.identities[1]
self.assertIsInstance(id1, Identity)
self.assertEqual(id1.id, None)
self.assertEqual(id1.name, 'John Smith')
self.assertEqual(id1.email, '*****@*****.**')
self.assertEqual(id1.username, 'jsmith')
self.assertEqual(id1.uuid, 'jsmith')
self.assertEqual(id1.source, 'unknown')
enrollments = uid.enrollments
enrollments.sort(key=lambda x: x.start)
self.assertEqual(len(enrollments), 2)
rol0 = enrollments[0]
self.assertIsInstance(rol0.organization, Organization)
self.assertEqual(rol0.organization.name, 'Example')
self.assertEqual(rol0.start, datetime.datetime(2010, 1, 1))
self.assertEqual(rol0.end, datetime.datetime(2011, 1, 1))
rol1 = enrollments[1]
self.assertIsInstance(rol1.organization, Organization)
self.assertEqual(rol1.organization.name, 'Bitergia')
# Active date for Bitergia is 2012-01-01, so this enrollment
# has that date instead of 2011-01-01
self.assertEqual(rol1.start, datetime.datetime(2012, 1, 1))
self.assertEqual(rol1.end, datetime.datetime(2100, 1, 1))
def test_not_valid_stream(self):
"""Check whether it prints an error when parsing invalid streams"""
with self.assertRaises(InvalidFormatError):
s = self.read_file(datadir('mailmap_invalid.txt'))
MailmapParser(s)
def test_valid_identities_already_exist(self):
"""Check method when an identity already exists but with distinct UUID"""
# The identity already exists but with a different UUID
uuid = api.add_identity(self.db, 'unknown', email='*****@*****.**')
api.add_identity(self.db,
source='scm',
email='*****@*****.**',
name='John Smith',
username='******',
uuid=uuid)
api.edit_profile(self.db,
uuid,
name='John Smith',
is_bot=False,
country_code='US')
parser = self.get_parser(datadir('sortinghat_valid.json'))
code = self.cmd.import_identities(parser)
self.assertEqual(code, CMD_SUCCESS)
# Check the contents of the registry
uids = api.unique_identities(self.db)
self.assertEqual(len(uids), 2)
# John Smith
uid = uids[1]
self.assertEqual(uid.uuid, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')
# The profile is updated because a new one was given
prf = uid.profile
self.assertEqual(prf.uuid, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')
self.assertEqual(prf.name, None)
self.assertEqual(prf.email, '*****@*****.**')
self.assertEqual(prf.gender, 'male')
self.assertEqual(prf.gender_acc, 100)
self.assertEqual(prf.is_bot, True)
self.assertEqual(prf.country, None)
ids = self.sort_identities(uid.identities)
self.assertEqual(len(ids), 3)
id0 = ids[0]
self.assertEqual(id0.id, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')
self.assertEqual(id0.name, None)
self.assertEqual(id0.email, '*****@*****.**')
self.assertEqual(id0.username, None)
self.assertEqual(id0.source, 'unknown')
id1 = ids[1]
self.assertEqual(id1.id, '880b3dfcb3a08712e5831bddc3dfe81fc5d7b331')
self.assertEqual(id1.name, 'John Smith')
self.assertEqual(id1.email, '*****@*****.**')
self.assertEqual(id1.username, None)
self.assertEqual(id1.source, 'scm')
id2 = ids[2]
self.assertEqual(id2.id, 'a9b403e150dd4af8953a52a4bb841051e4b705d9')
self.assertEqual(id2.name, 'John Smith')
self.assertEqual(id2.email, '*****@*****.**')
self.assertEqual(id2.username, 'jsmith')
self.assertEqual(id2.source, 'scm')
def test_reset(self):
"""Check if stored relationships and enrollments are removed before loading"""
# These identities will be split and enrollments removed
uuid = api.add_identity(self.db, 'unknown', email='*****@*****.**')
api.add_identity(self.db,
source='scm',
email='*****@*****.**',
name='John Smith',
username='******',
uuid=uuid)
api.add_organization(self.db, 'LibreSoft')
api.add_enrollment(self.db, uuid, 'LibreSoft',
datetime.datetime(2000, 1, 1, 0, 0),
datetime.datetime(2100, 1, 1, 0, 0))
parser = self.get_parser(datadir('sortinghat_valid.json'))
code = self.cmd.import_identities(parser, reset=True)
self.assertEqual(code, CMD_SUCCESS)
# Check the contents of the registry
uids = api.unique_identities(self.db)
self.assertEqual(len(uids), 3)
# Jane Roe
uid = uids[0]
self.assertEqual(uid.uuid, '17ab00ed3825ec2f50483e33c88df223264182ba')
ids = self.sort_identities(uid.identities)
self.assertEqual(len(ids), 3)
id0 = ids[0]
self.assertEqual(id0.id, '17ab00ed3825ec2f50483e33c88df223264182ba')
self.assertEqual(id0.name, 'Jane Roe')
self.assertEqual(id0.email, '*****@*****.**')
self.assertEqual(id0.username, 'jroe')
self.assertEqual(id0.source, 'scm')
enrollments = api.enrollments(self.db, uid.uuid)
self.assertEqual(len(enrollments), 3)
# [email protected]
uid = uids[1]
self.assertEqual(uid.uuid, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')
ids = self.sort_identities(uid.identities)
self.assertEqual(len(ids), 1)
id0 = ids[0]
self.assertEqual(id0.id, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')
self.assertEqual(id0.name, None)
self.assertEqual(id0.email, '*****@*****.**')
self.assertEqual(id0.username, None)
self.assertEqual(id0.source, 'unknown')
enrollments = api.enrollments(self.db, uid.uuid)
self.assertEqual(len(enrollments), 0)
# John Smith
uid = uids[2]
self.assertEqual(uid.uuid, 'a9b403e150dd4af8953a52a4bb841051e4b705d9')
ids = self.sort_identities(uid.identities)
self.assertEqual(len(ids), 2)
id1 = ids[0]
self.assertEqual(id1.id, '880b3dfcb3a08712e5831bddc3dfe81fc5d7b331')
self.assertEqual(id1.name, 'John Smith')
self.assertEqual(id1.email, '*****@*****.**')
self.assertEqual(id1.username, None)
self.assertEqual(id1.source, 'scm')
id2 = ids[1]
self.assertEqual(id2.id, 'a9b403e150dd4af8953a52a4bb841051e4b705d9')
self.assertEqual(id2.name, 'John Smith')
self.assertEqual(id2.email, '*****@*****.**')
self.assertEqual(id2.username, 'jsmith')
self.assertEqual(id2.source, 'scm')
enrollments = api.enrollments(self.db, uid.uuid)
self.assertEqual(len(enrollments), 1)
def test_import_to_non_empty_registry(self):
"""Test load (and overwrite) process in a registry with some contents"""
# First, load some data
api.add_organization(self.db, 'Example')
api.add_domain(self.db, 'Example', 'example.com')
api.add_organization(self.db, 'Bitergia')
api.add_domain(self.db, 'Bitergia', 'bitergia.net')
api.add_domain(self.db, 'Bitergia', 'bitergia.com')
# Import new data, overwriting existing relationships
parser = self.get_parser(datadir('sortinghat_orgs_valid_alt.json'))
self.cmd.import_organizations(parser, True)
# Check the contents of the registry
orgs = api.registry(self.db)
self.assertEqual(len(orgs), 4)
# Bitergia
org = orgs[0]
self.assertEqual(org.name, 'Bitergia')
doms = org.domains
self.assertEqual(len(doms), 4)
dom = doms[0]
self.assertEqual(dom.domain, 'api.bitergia.com')
self.assertEqual(dom.is_top_domain, False)
dom = doms[1]
self.assertEqual(dom.domain, 'bitergia.com')
self.assertEqual(dom.is_top_domain, True)
dom = doms[2]
self.assertEqual(dom.domain, 'bitergia.net')
self.assertEqual(dom.is_top_domain, True)
dom = doms[3]
self.assertEqual(dom.domain, 'test.bitergia.com')
self.assertEqual(dom.is_top_domain, False)
# Example
org = orgs[1]
self.assertEqual(org.name, 'Example')
doms = org.domains
self.assertEqual(len(doms), 1)
dom = doms[0]
self.assertEqual(dom.domain, 'example.net')
self.assertEqual(dom.is_top_domain, True)
# GSyC/LibreSoft
org = orgs[2]
self.assertEqual(org.name, 'GSyC/LibreSoft')
doms = org.domains
self.assertEqual(len(doms), 2)
dom = doms[0]
self.assertEqual(dom.domain, 'example.com')
self.assertEqual(dom.is_top_domain, True)
dom = doms[1]
self.assertEqual(dom.domain, 'libresoft.es')
self.assertEqual(dom.is_top_domain, True)
# Unknown
org = orgs[3]
self.assertEqual(org.name, 'Unknown')
doms = org.domains
self.assertEqual(len(doms), 0)
def test_enrollments_parser(self):
aliases = self.read_file(datadir('gitdm_email_aliases_valid.txt'))
email_to_employer = self.read_file(
datadir('gitdm_email_to_employer_valid.txt'))
parser = GitdmParser(aliases=aliases,
email_to_employer=email_to_employer,
source='unknown')
# Parsed unique identities
uids = parser.identities
self.assertEqual(len(uids), 4)
# [email protected] & [email protected]
uid = uids[0]
self.assertIsInstance(uid, UniqueIdentity)
self.assertEqual(uid.uuid, '*****@*****.**')
self.assertIsInstance(uid, UniqueIdentity)
ids = uid.identities
self.assertEqual(len(ids), 2)
id0 = ids[0]
self.assertIsInstance(id0, Identity)
self.assertEqual(id0.name, None)
self.assertEqual(id0.email, '*****@*****.**')
self.assertEqual(id0.username, None)
self.assertEqual(id0.source, 'unknown')
self.assertEqual(id0.uuid, None)
id1 = ids[1]
self.assertIsInstance(id1, Identity)
self.assertEqual(id1.name, None)
self.assertEqual(id1.email, '*****@*****.**')
self.assertEqual(id1.username, None)
self.assertEqual(id1.source, 'unknown')
self.assertEqual(id1.uuid, None)
enrollments = uid.enrollments
enrollments.sort(key=lambda x: x.organization.name)
self.assertEqual(len(uid.enrollments), 2)
rol = uid.enrollments[0]
self.assertIsInstance(rol, Enrollment)
self.assertEqual(rol.organization.name, 'Example Company')
self.assertEqual(rol.start, datetime.datetime(1900, 1, 1, 0, 0))
self.assertEqual(rol.end, datetime.datetime(2100, 1, 1, 0, 0))
rol = uid.enrollments[1]
self.assertIsInstance(rol, Enrollment)
self.assertEqual(rol.organization.name, 'LibreSoft')
self.assertEqual(rol.start, datetime.datetime(1900, 1, 1, 0, 0))
self.assertEqual(rol.end, datetime.datetime(2100, 1, 1, 0, 0))
# [email protected] & [email protected]
uid = uids[1]
self.assertIsInstance(uid, UniqueIdentity)
self.assertEqual(uid.uuid, '*****@*****.**')
ids = uid.identities
self.assertEqual(len(ids), 2)
id0 = ids[0]
self.assertIsInstance(id0, Identity)
self.assertEqual(id0.name, None)
self.assertEqual(id0.email, '*****@*****.**')
self.assertEqual(id0.username, None)
self.assertEqual(id0.source, 'unknown')
self.assertEqual(id0.uuid, None)
id0 = ids[1]
self.assertIsInstance(id0, Identity)
self.assertEqual(id0.name, None)
self.assertEqual(id0.email, '*****@*****.**')
self.assertEqual(id0.username, None)
self.assertEqual(id0.source, 'unknown')
self.assertEqual(id0.uuid, None)
self.assertEqual(len(uid.enrollments), 1)
rol = uid.enrollments[0]
self.assertIsInstance(rol, Enrollment)
self.assertEqual(rol.organization.name, 'Bitergia')
self.assertEqual(rol.start, datetime.datetime(1900, 1, 1, 0, 0))
self.assertEqual(rol.end, datetime.datetime(2100, 1, 1, 0, 0))
# jrae@laptop & jrae@mylaptop
uid = uids[2]
self.assertIsInstance(uid, UniqueIdentity)
self.assertEqual(uid.uuid, 'jrae@mylaptop')
ids = uid.identities
self.assertEqual(len(ids), 2)
id0 = ids[0]
self.assertIsInstance(id0, Identity)
self.assertEqual(id0.name, None)
self.assertEqual(id0.email, None)
self.assertEqual(id0.username, 'jrae@mylaptop')
self.assertEqual(id0.source, 'unknown')
self.assertEqual(id0.uuid, None)
id0 = ids[1]
self.assertIsInstance(id0, Identity)
self.assertEqual(id0.name, None)
self.assertEqual(id0.email, None)
self.assertEqual(id0.username, 'jrae@laptop')
self.assertEqual(id0.source, 'unknown')
self.assertEqual(id0.uuid, None)
self.assertEqual(len(uid.enrollments), 0)
# [email protected]
uid = uids[3]
self.assertIsInstance(uid, UniqueIdentity)
self.assertEqual(uid.uuid, '*****@*****.**')
ids = uid.identities
self.assertEqual(len(ids), 1)
id0 = ids[0]
self.assertIsInstance(id0, Identity)
self.assertEqual(id0.name, None)
self.assertEqual(id0.email, '*****@*****.**')
self.assertEqual(id0.username, None)
self.assertEqual(id0.source, 'unknown')
self.assertEqual(id0.uuid, None)
self.assertEqual(len(uid.enrollments), 2)
rol = uid.enrollments[0]
self.assertIsInstance(rol, Enrollment)
self.assertEqual(rol.organization.name, 'Bitergia')
self.assertEqual(rol.start, datetime.datetime(1900, 1, 1, 0, 0))
self.assertEqual(rol.end, datetime.datetime(2015, 1, 1, 0, 0))
rol = uid.enrollments[1]
self.assertIsInstance(rol, Enrollment)
self.assertEqual(rol.organization.name, 'Example Company')
self.assertEqual(rol.start, datetime.datetime(2015, 1, 1, 0, 0))
self.assertEqual(rol.end, datetime.datetime(2100, 1, 1, 0, 0))
# Parsed organizations
orgs = parser.organizations
self.assertEqual(len(orgs), 3)
org = orgs[0]
self.assertIsInstance(org, Organization)
self.assertEqual(org.name, 'Bitergia')
org = orgs[1]
self.assertIsInstance(org, Organization)
self.assertEqual(org.name, 'Example Company')
org = orgs[2]
self.assertIsInstance(org, Organization)
self.assertEqual(org.name, 'LibreSoft')
def test_organizations_parser(self):
"""Check whether it parses a valid organizations file"""
stream = self.read_file(datadir('gitdm_orgs_valid.txt'))
parser = GitdmParser(domain_to_employer=stream)
# Parsed unique identities
uids = parser.identities
self.assertEqual(len(uids), 0)
# Parsed organizations
orgs = parser.organizations
self.assertEqual(len(orgs), 3)
# Bitergia entries
org = orgs[0]
self.assertIsInstance(org, Organization)
self.assertEqual(org.name, 'Bitergia')
doms = org.domains
self.assertEqual(len(doms), 3)
dom = doms[0]
self.assertIsInstance(dom, Domain)
self.assertEqual(dom.domain, 'bitergia.com')
self.assertEqual(dom.is_top_domain, False)
dom = doms[1]
self.assertIsInstance(dom, Domain)
self.assertEqual(dom.domain, 'bitergia.net')
self.assertEqual(dom.is_top_domain, False)
dom = doms[2]
self.assertIsInstance(dom, Domain)
self.assertEqual(dom.domain, 'example.com')
self.assertEqual(dom.is_top_domain, False)
# Example entries
org = orgs[1]
self.assertIsInstance(org, Organization)
self.assertEqual(org.name, 'Example')
doms = org.domains
self.assertEqual(len(doms), 3)
dom = doms[0]
self.assertIsInstance(dom, Domain)
self.assertEqual(dom.domain, 'example.com')
self.assertEqual(dom.is_top_domain, False)
dom = doms[1]
self.assertIsInstance(dom, Domain)
self.assertEqual(dom.domain, 'example.org')
self.assertEqual(dom.is_top_domain, False)
dom = doms[2]
self.assertIsInstance(dom, Domain)
self.assertEqual(dom.domain, 'example.net')
self.assertEqual(dom.is_top_domain, False)
# GSyC/Libresof entries
org = orgs[2]
self.assertIsInstance(org, Organization)
self.assertEqual(org.name, 'GSyC/LibreSoft')
doms = org.domains
self.assertEqual(len(doms), 2)
dom = doms[0]
self.assertIsInstance(dom, Domain)
self.assertEqual(dom.domain, 'libresoft.es')
self.assertEqual(dom.is_top_domain, False)
dom = doms[1]
self.assertIsInstance(dom, Domain)
self.assertEqual(dom.domain, 'gsyc.es')
self.assertEqual(dom.is_top_domain, False)
def test_aliases_parser(self):
aliases = self.read_file(datadir('gitdm_email_aliases_valid.txt'))
parser = GitdmParser(aliases=aliases)
# Parsed unique identities
uids = parser.identities
self.assertEqual(len(uids), 3)
# [email protected] & [email protected]
uid = uids[0]
self.assertIsInstance(uid, UniqueIdentity)
self.assertEqual(uid.uuid, '*****@*****.**')
self.assertIsInstance(uid, UniqueIdentity)
ids = uid.identities
self.assertEqual(len(ids), 2)
id0 = ids[0]
self.assertIsInstance(id0, Identity)
self.assertEqual(id0.name, None)
self.assertEqual(id0.email, '*****@*****.**')
self.assertEqual(id0.username, None)
self.assertEqual(id0.source, 'gitdm')
self.assertEqual(id0.uuid, None)
id1 = ids[1]
self.assertIsInstance(id1, Identity)
self.assertEqual(id1.name, None)
self.assertEqual(id1.email, '*****@*****.**')
self.assertEqual(id1.username, None)
self.assertEqual(id1.source, 'gitdm')
self.assertEqual(id1.uuid, None)
self.assertEqual(len(uid.enrollments), 0)
# [email protected] & [email protected]
uid = uids[1]
self.assertIsInstance(uid, UniqueIdentity)
self.assertEqual(uid.uuid, '*****@*****.**')
ids = uid.identities
self.assertEqual(len(ids), 2)
id0 = ids[0]
self.assertIsInstance(id0, Identity)
self.assertEqual(id0.name, None)
self.assertEqual(id0.email, '*****@*****.**')
self.assertEqual(id0.username, None)
self.assertEqual(id0.source, 'gitdm')
self.assertEqual(id0.uuid, None)
id0 = ids[1]
self.assertIsInstance(id0, Identity)
self.assertEqual(id0.name, None)
self.assertEqual(id0.email, '*****@*****.**')
self.assertEqual(id0.username, None)
self.assertEqual(id0.source, 'gitdm')
self.assertEqual(id0.uuid, None)
self.assertEqual(len(uid.enrollments), 0)
# jrae@laptop & jrae@mylaptop
uid = uids[2]
self.assertIsInstance(uid, UniqueIdentity)
self.assertEqual(uid.uuid, 'jrae@mylaptop')
ids = uid.identities
self.assertEqual(len(ids), 2)
id0 = ids[0]
self.assertIsInstance(id0, Identity)
self.assertEqual(id0.name, None)
self.assertEqual(id0.email, None)
self.assertEqual(id0.username, 'jrae@mylaptop')
self.assertEqual(id0.source, 'gitdm')
self.assertEqual(id0.uuid, None)
id0 = ids[1]
self.assertIsInstance(id0, Identity)
self.assertEqual(id0.name, None)
self.assertEqual(id0.email, None)
self.assertEqual(id0.username, 'jrae@laptop')
self.assertEqual(id0.source, 'gitdm')
self.assertEqual(id0.uuid, None)
self.assertEqual(len(uid.enrollments), 0)
def test_valid_identities_stream(self):
"""Check insertion of valid data from a file"""
stream = self.read_file(datadir('stackalytics_valid.json'))
parser = StackalyticsParser(stream, source='unknown')
uids = parser.identities
# Check parsed identities
self.assertEqual(len(uids), 2)
# John Doe unique identity
uid = uids[0]
self.assertIsInstance(uid, UniqueIdentity)
self.assertEqual(uid.uuid, 'John Doe')
self.assertEqual(len(uid.identities), 3)
id0 = uid.identities[0]
self.assertIsInstance(id0, Identity)
self.assertEqual(id0.id, None)
self.assertEqual(id0.name, 'John Doe')
self.assertEqual(id0.email, None)
self.assertEqual(id0.username, None)
self.assertEqual(id0.uuid, 'John Doe')
self.assertEqual(id0.source, 'unknown')
enrollments = uid.enrollments
self.assertEqual(len(enrollments), 2)
rol0 = enrollments[0]
self.assertIsInstance(rol0, Enrollment)
self.assertIsInstance(rol0.organization, Organization)
self.assertEqual(rol0.organization.name, 'Bitergia')
self.assertEqual(rol0.start, MIN_PERIOD_DATE)
self.assertEqual(rol0.end, MAX_PERIOD_DATE)
rol1 = enrollments[1]
self.assertIsInstance(rol1, Enrollment)
self.assertIsInstance(rol1.organization, Organization)
self.assertEqual(rol1.organization.name, 'Example')
self.assertEqual(rol1.start, MIN_PERIOD_DATE)
self.assertEqual(rol1.end, datetime.datetime(2010, 1, 1, 0, 0, 0))
# John Smith unique identity
uid = uids[1]
self.assertIsInstance(uid, UniqueIdentity)
self.assertEqual(uid.uuid, 'John Smith')
self.assertEqual(len(uid.identities), 4)
id0 = uid.identities[0]
self.assertIsInstance(id0, Identity)
self.assertEqual(id0.id, None)
self.assertEqual(id0.name, 'John Smith')
self.assertEqual(id0.email, None)
self.assertEqual(id0.username, None)
self.assertEqual(id0.uuid, 'John Smith')
self.assertEqual(id0.source, 'unknown')
id1 = uid.identities[1]
self.assertIsInstance(id1, Identity)
self.assertEqual(id1.id, None)
self.assertEqual(id1.name, 'John Smith')
self.assertEqual(id1.email, '*****@*****.**')
self.assertEqual(id1.username, None)
self.assertEqual(id1.uuid, 'John Smith')
self.assertEqual(id1.source, 'unknown')
id2 = uid.identities[2]
self.assertIsInstance(id2, Identity)
self.assertEqual(id2.id, None)
self.assertEqual(id2.name, 'John Smith')
self.assertEqual(id2.email, None)
self.assertEqual(id2.username, 'jsmith')
self.assertEqual(id2.uuid, 'John Smith')
self.assertEqual(id2.source, 'unknown:gerrit')
id3 = uid.identities[3]
self.assertIsInstance(id3, Identity)
self.assertEqual(id3.id, None)
self.assertEqual(id3.name, 'John Smith')
self.assertEqual(id3.email, None)
self.assertEqual(id3.username, '0-jsmith')
self.assertEqual(id3.uuid, 'John Smith')
self.assertEqual(id3.source, 'unknown:launchpad')
enrollments = uid.enrollments
self.assertEqual(len(enrollments), 1)
rol0 = enrollments[0]
self.assertIsInstance(rol0, Enrollment)
self.assertIsInstance(rol0.organization, Organization)
self.assertEqual(rol0.organization.name, 'Example')
self.assertEqual(rol0.start, MIN_PERIOD_DATE)
self.assertEqual(rol0.end, MAX_PERIOD_DATE)
def test_valid_organizations_stream(self):
"""Check parsed orgs and identities from a valid file"""
stream = self.read_file(datadir('mailmap_orgs.txt'))
parser = MailmapParser(stream, has_orgs=True, source='unknown')
# Check parsed organizations
# Unaffiliated was not included because it is not considered
# as an organization
orgs = parser.organizations
self.assertEqual(len(orgs), 2)
org = orgs[0]
self.assertIsInstance(org, Organization)
self.assertEqual(org.name, 'Bitergia')
self.assertEqual(len(org.domains), 0)
org = orgs[1]
self.assertIsInstance(org, Organization)
self.assertEqual(org.name, 'Example Inc.')
self.assertEqual(len(org.domains), 0)
# Check identities and enrollments
uids = parser.identities
self.assertEqual(len(uids), 3)
# John Doe identity
uid = uids[0]
self.assertIsInstance(uid, UniqueIdentity)
self.assertEqual(uid.uuid, '*****@*****.**')
self.assertEqual(len(uid.identities), 1)
id0 = uid.identities[0]
self.assertIsInstance(id0, Identity)
self.assertEqual(id0.id, None)
self.assertEqual(id0.name, 'John Doe')
self.assertEqual(id0.email, '*****@*****.**')
self.assertEqual(id0.username, None)
self.assertEqual(id0.uuid, '*****@*****.**')
self.assertEqual(id0.source, 'unknown')
enrollments = uid.enrollments
self.assertEqual(len(enrollments), 1)
rol = enrollments[0]
self.assertIsInstance(rol.organization, Organization)
self.assertEqual(rol.organization.name, 'Bitergia')
self.assertEqual(rol.start, datetime.datetime(1900, 1, 1, 0, 0, 0))
self.assertEqual(rol.end, datetime.datetime(2100, 1, 1, 0, 0, 0))
# Jane Rae identity
uid = uids[1]
self.assertIsInstance(uid, UniqueIdentity)
self.assertEqual(uid.uuid, '*****@*****.**')
self.assertEqual(len(uid.identities), 1)
id0 = uid.identities[0]
self.assertIsInstance(id0, Identity)
self.assertEqual(id0.id, None)
self.assertEqual(id0.name, 'Jane Rae')
self.assertEqual(id0.email, '*****@*****.**')
self.assertEqual(id0.username, None)
self.assertEqual(id0.uuid, '*****@*****.**')
self.assertEqual(id0.source, 'unknown')
enrollments = uid.enrollments
self.assertEqual(len(enrollments), 0)
# John Smith identity
uid = uids[2]
self.assertIsInstance(uid, UniqueIdentity)
self.assertEqual(uid.uuid, '*****@*****.**')
self.assertEqual(len(uid.identities), 1)
id0 = uid.identities[0]
self.assertIsInstance(id0, Identity)
self.assertEqual(id0.id, None)
self.assertEqual(id0.name, 'John Smith')
self.assertEqual(id0.email, '*****@*****.**')
self.assertEqual(id0.username, None)
self.assertEqual(id0.uuid, '*****@*****.**')
self.assertEqual(id0.source, 'unknown')
enrollments = uid.enrollments
self.assertEqual(len(enrollments), 2)
rol = enrollments[0]
self.assertIsInstance(rol.organization, Organization)
self.assertEqual(rol.organization.name, 'Bitergia')
self.assertEqual(rol.start, datetime.datetime(1900, 1, 1, 0, 0, 0))
self.assertEqual(rol.end, datetime.datetime(2100, 1, 1, 0, 0, 0))
rol = enrollments[1]
self.assertIsInstance(rol.organization, Organization)
self.assertEqual(rol.organization.name, 'Example Inc.')
self.assertEqual(rol.start, datetime.datetime(1900, 1, 1, 0, 0, 0))
self.assertEqual(rol.end, datetime.datetime(2100, 1, 1, 0, 0, 0))
def test_match_new_identities(self):
"""Check whether it matches only new identities"""
parser = self.get_parser(datadir('sortinghat_valid.json'))
code = self.cmd.import_identities(parser, matching='default')
self.assertEqual(code, CMD_SUCCESS)
# Check the contents of the registry
uids = api.unique_identities(self.db)
self.assertEqual(len(uids), 2)
# We add a new identity that matches with some of the
# already inserted
jsmith_uuid = api.add_identity(self.db,
'unknown',
email='*****@*****.**')
uids = api.unique_identities(self.db)
self.assertEqual(len(uids), 3)
# This file has a new identity, only Jane Roe will match.
parser = self.get_parser(datadir('sortinghat_valid_updated.json'))
code = self.cmd.import_identities(parser,
matching='default',
match_new=True)
self.assertEqual(code, CMD_SUCCESS)
uids = api.unique_identities(self.db)
self.assertEqual(len(uids), 3)
# Jane Roe
uid = uids[0]
self.assertEqual(uid.uuid, '17ab00ed3825ec2f50483e33c88df223264182ba')
ids = self.sort_identities(uid.identities)
self.assertEqual(len(ids), 4)
self.assertEqual(ids[0].id, '17ab00ed3825ec2f50483e33c88df223264182ba')
self.assertEqual(ids[1].id, '22d1b20763c6f5822bdda8508957486c547bb9de')
self.assertEqual(ids[2].id, '322397ed782a798ffd9d0bc7e293df4292fe075d')
self.assertEqual(ids[3].id, '8ff87accaf518070bdb494b87f4d5a10e7605b47')
# Now, if we reload again the file but setting 'match_new' to false,
# the identity that we inserted before will match "John Smith"
parser = self.get_parser(datadir('sortinghat_valid_updated.json'))
code = self.cmd.import_identities(parser, matching='default')
self.assertEqual(code, CMD_SUCCESS)
uids = api.unique_identities(self.db)
self.assertEqual(len(uids), 2)
# John Smith
uid = uids[1]
self.assertEqual(uid.uuid, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')
ids = self.sort_identities(uid.identities)
self.assertEqual(len(ids), 3)
self.assertEqual(ids[0].id, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')
self.assertEqual(ids[0].id, jsmith_uuid)
self.assertEqual(ids[1].id, '880b3dfcb3a08712e5831bddc3dfe81fc5d7b331')
self.assertEqual(ids[2].id, 'a9b403e150dd4af8953a52a4bb841051e4b705d9')
def test_valid_identities_stream(self):
"""Check parsed identities from a valid file"""
stream = self.read_file(datadir('mailmap_identities.txt'))
parser = MailmapParser(stream, source='unknown')
# Check parsed identities
uids = parser.identities
self.assertEqual(len(uids), 3)
# John Doe identity
uid = uids[0]
self.assertIsInstance(uid, UniqueIdentity)
self.assertEqual(uid.uuid, '*****@*****.**')
self.assertEqual(len(uid.identities), 3)
id_ = uid.identities[0]
self.assertIsInstance(id_, Identity)
self.assertEqual(id_.id, None)
self.assertEqual(id_.name, 'John Doe')
self.assertEqual(id_.email, '*****@*****.**')
self.assertEqual(id_.username, None)
self.assertEqual(id_.uuid, '*****@*****.**')
self.assertEqual(id_.source, 'unknown')
id_ = uid.identities[1]
self.assertIsInstance(id_, Identity)
self.assertEqual(id_.id, None)
self.assertEqual(id_.name, 'John Doe')
self.assertEqual(id_.email, '*****@*****.**')
self.assertEqual(id_.username, None)
self.assertEqual(id_.uuid, '*****@*****.**')
self.assertEqual(id_.source, 'unknown')
id_ = uid.identities[2]
self.assertIsInstance(id_, Identity)
self.assertEqual(id_.id, None)
self.assertEqual(id_.name, 'Doe J')
self.assertEqual(id_.email, '*****@*****.**')
self.assertEqual(id_.username, None)
self.assertEqual(id_.uuid, '*****@*****.**')
self.assertEqual(id_.source, 'unknown')
self.assertEqual(len(uid.enrollments), 0)
# Jane Rae identity
uid = uids[1]
self.assertIsInstance(uid, UniqueIdentity)
self.assertEqual(uid.uuid, '*****@*****.**')
self.assertEqual(len(uid.identities), 3)
id_ = uid.identities[0]
self.assertIsInstance(id_, Identity)
self.assertEqual(id_.id, None)
self.assertEqual(id_.name, 'Jane Rae')
self.assertEqual(id_.email, '*****@*****.**')
self.assertEqual(id_.username, None)
self.assertEqual(id_.uuid, '*****@*****.**')
self.assertEqual(id_.source, 'unknown')
id_ = uid.identities[1]
self.assertIsInstance(id_, Identity)
self.assertEqual(id_.id, None)
self.assertEqual(id_.name, 'Jane R. Rae')
self.assertEqual(id_.email, '*****@*****.**')
self.assertEqual(id_.username, None)
self.assertEqual(id_.uuid, '*****@*****.**')
self.assertEqual(id_.source, 'unknown')
id_ = uid.identities[2]
self.assertIsInstance(id_, Identity)
self.assertEqual(id_.id, None)
self.assertEqual(id_.name, 'Jane')
self.assertEqual(id_.email, '*****@*****.**')
self.assertEqual(id_.username, None)
self.assertEqual(id_.uuid, '*****@*****.**')
self.assertEqual(id_.source, 'unknown')
self.assertEqual(len(uid.enrollments), 0)
# Jonh Smith identity
uid = uids[2]
self.assertIsInstance(uid, UniqueIdentity)
self.assertEqual(uid.uuid, '*****@*****.**')
self.assertEqual(len(uid.identities), 2)
id_ = uid.identities[0]
self.assertIsInstance(id_, Identity)
self.assertEqual(id_.id, None)
self.assertEqual(id_.name, 'John Smith')
self.assertEqual(id_.email, '*****@*****.**')
self.assertEqual(id_.username, None)
self.assertEqual(id_.uuid, '*****@*****.**')
self.assertEqual(id_.source, 'unknown')
id_ = uid.identities[1]
self.assertIsInstance(id_, Identity)
self.assertEqual(id_.id, None)
self.assertEqual(id_.name, 'John Smith')
self.assertEqual(id_.email, '*****@*****.**')
self.assertEqual(id_.username, None)
self.assertEqual(id_.uuid, '*****@*****.**')
self.assertEqual(id_.source, 'unknown')
# No organizations were parsed
orgs = parser.organizations
self.assertEqual(len(orgs), 0)
def test_valid_identities_with_default_matching(self):
"""Check insertion, matching and merging of valid data"""
# First, insert the identity that will match with one
# from the file
api.add_organization(self.db, 'Example')
uuid = api.add_identity(self.db, 'unknown', email='*****@*****.**')
api.add_enrollment(self.db, uuid, 'Example',
datetime.datetime(2000, 1, 1, 0, 0),
datetime.datetime(2100, 1, 1, 0, 0))
api.edit_profile(self.db,
uuid,
name='John Smith',
is_bot=False,
country_code='US')
parser = self.get_parser(datadir('sortinghat_valid.json'))
code = self.cmd.import_identities(parser, matching='default')
self.assertEqual(code, CMD_SUCCESS)
# Check the contents of the registry
uids = api.unique_identities(self.db)
self.assertEqual(len(uids), 2)
# Jane Roe
uid = uids[0]
self.assertEqual(uid.uuid, '17ab00ed3825ec2f50483e33c88df223264182ba')
prf = uid.profile
self.assertEqual(prf.uuid, '17ab00ed3825ec2f50483e33c88df223264182ba')
self.assertEqual(prf.name, 'Jane Roe')
self.assertEqual(prf.email, '*****@*****.**')
self.assertEqual(prf.gender, None)
self.assertEqual(prf.gender_acc, None)
self.assertEqual(prf.is_bot, False)
self.assertEqual(prf.country_code, 'US')
self.assertEqual(prf.country.alpha3, 'USA')
self.assertEqual(prf.country.code, 'US')
self.assertEqual(prf.country.name, 'United States of America')
ids = self.sort_identities(uid.identities)
self.assertEqual(len(ids), 3)
id0 = ids[0]
self.assertEqual(id0.id, '17ab00ed3825ec2f50483e33c88df223264182ba')
self.assertEqual(id0.name, 'Jane Roe')
self.assertEqual(id0.email, '*****@*****.**')
self.assertEqual(id0.username, 'jroe')
self.assertEqual(id0.source, 'scm')
id1 = ids[1]
self.assertEqual(id1.id, '22d1b20763c6f5822bdda8508957486c547bb9de')
self.assertEqual(id1.name, None)
self.assertEqual(id1.email, '*****@*****.**')
self.assertEqual(id1.username, None)
self.assertEqual(id1.source, 'unknown')
id2 = ids[2]
self.assertEqual(id2.id, '322397ed782a798ffd9d0bc7e293df4292fe075d')
self.assertEqual(id2.name, None)
self.assertEqual(id2.email, '*****@*****.**')
self.assertEqual(id2.username, None)
self.assertEqual(id2.source, 'scm')
enrollments = api.enrollments(self.db, uid.uuid)
self.assertEqual(len(enrollments), 3)
# John Smith
uid = uids[1]
self.assertEqual(uid.uuid, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')
ids = self.sort_identities(uid.identities)
self.assertEqual(len(ids), 3)
# The profile was merged
prf = uid.profile
self.assertEqual(prf.uuid, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')
self.assertEqual(prf.name, 'John Smith')
self.assertEqual(prf.email, '*****@*****.**')
self.assertEqual(prf.gender, 'male')
self.assertEqual(prf.gender_acc, 100)
self.assertEqual(prf.is_bot, True)
self.assertEqual(prf.country_code, 'US')
self.assertEqual(prf.country.code, 'US')
self.assertEqual(prf.country.name, 'United States of America')
id0 = ids[0]
self.assertEqual(id0.id, '2371a34a0ac65fbd9d631464ee41d583ec0e1e88')
self.assertEqual(id0.name, None)
self.assertEqual(id0.email, '*****@*****.**')
self.assertEqual(id0.username, None)
self.assertEqual(id0.source, 'unknown')
id1 = ids[1]
self.assertEqual(id1.id, '880b3dfcb3a08712e5831bddc3dfe81fc5d7b331')
self.assertEqual(id1.name, 'John Smith')
self.assertEqual(id1.email, '*****@*****.**')
self.assertEqual(id1.username, None)
self.assertEqual(id1.source, 'scm')
id2 = ids[2]
self.assertEqual(id2.id, 'a9b403e150dd4af8953a52a4bb841051e4b705d9')
self.assertEqual(id2.name, 'John Smith')
self.assertEqual(id2.email, '*****@*****.**')
self.assertEqual(id2.username, 'jsmith')
self.assertEqual(id2.source, 'scm')
# Enrollments were merged
enrollments = api.enrollments(self.db, uid.uuid)
self.assertEqual(len(enrollments), 1)
rol0 = enrollments[0]
self.assertEqual(rol0.organization.name, 'Example')
self.assertEqual(rol0.start, datetime.datetime(2000, 1, 1, 0, 0))
self.assertEqual(rol0.end, datetime.datetime(2100, 1, 1, 0, 0))
import os.path
import sys
import unittest
import configparser
if '..' not in sys.path:
sys.path.insert(0, '..')
from sortinghat.command import CMD_SUCCESS
from sortinghat.cmd.config import Config
from tests.base import TestCommandCaseBase, datadir
MOCK_CONFIG_FILE = datadir('mock_config_file.cfg')
INVALID_CONFIG_FILE = "Configuration file not given"
SET_KEY_CONFIG_ERROR = "%(param)s parameter does not exists or cannot be set"
GET_KEY_CONFIG_ERROR = "%(param)s parameter does not exists"
NOT_FOUND_FILE_ERROR = "./data/invalid_config_file.cfg config file does not exist"
class TestConfigCaseBase(TestCommandCaseBase):
"""Defines common setup and teardown methods on config unit tests"""
cmd_klass = Config
def load_test_dataset(self):
pass
def test_valid_identities_file(self):
"""Check insertion of valid data from a file"""
parser = self.get_parser(datadir('sortinghat_valid.json'))
code = self.cmd.import_identities(parser)
self.assertEqual(code, CMD_SUCCESS)
# Check the contents of the registry
uids = api.unique_identities(self.db)
self.assertEqual(len(uids), 2)
# Jane Roe
uid = uids[0]
self.assertEqual(uid.uuid, '17ab00ed3825ec2f50483e33c88df223264182ba')
prf = uid.profile
self.assertEqual(prf.uuid, '17ab00ed3825ec2f50483e33c88df223264182ba')
self.assertEqual(prf.name, 'Jane Roe')
self.assertEqual(prf.email, '*****@*****.**')
self.assertEqual(prf.gender, None)
self.assertEqual(prf.gender_acc, None)
self.assertEqual(prf.is_bot, False)
self.assertEqual(prf.country_code, 'US')
self.assertEqual(prf.country.alpha3, 'USA')
self.assertEqual(prf.country.code, 'US')
self.assertEqual(prf.country.name, 'United States of America')
ids = self.sort_identities(uid.identities)
self.assertEqual(len(ids), 3)
id0 = ids[0]
self.assertEqual(id0.id, '17ab00ed3825ec2f50483e33c88df223264182ba')
self.assertEqual(id0.name, 'Jane Roe')
self.assertEqual(id0.email, '*****@*****.**')
self.assertEqual(id0.username, 'jroe')
self.assertEqual(id0.source, 'scm')
id1 = ids[1]
self.assertEqual(id1.id, '22d1b20763c6f5822bdda8508957486c547bb9de')
self.assertEqual(id1.name, None)
self.assertEqual(id1.email, '*****@*****.**')
self.assertEqual(id1.username, None)
self.assertEqual(id1.source, 'unknown')
id2 = ids[2]
self.assertEqual(id2.id, '322397ed782a798ffd9d0bc7e293df4292fe075d')
self.assertEqual(id2.name, None)
self.assertEqual(id2.email, '*****@*****.**')
self.assertEqual(id2.username, None)
self.assertEqual(id2.source, 'scm')
enrollments = api.enrollments(self.db, uid.uuid)
self.assertEqual(len(enrollments), 3)
rol0 = enrollments[0]
self.assertEqual(rol0.organization.name, 'Bitergia')
self.assertEqual(rol0.start, datetime.datetime(1999, 1, 1, 0, 0))
self.assertEqual(rol0.end, datetime.datetime(2000, 1, 1, 0, 0))
rol1 = enrollments[1]
self.assertEqual(rol1.organization.name, 'Bitergia')
self.assertEqual(rol1.start, datetime.datetime(2006, 1, 1, 0, 0))
self.assertEqual(rol1.end, datetime.datetime(2008, 1, 1, 0, 0))
rol2 = enrollments[2]
self.assertEqual(rol2.organization.name, 'Example')
self.assertEqual(rol2.start, datetime.datetime(1900, 1, 1, 0, 0))
self.assertEqual(rol2.end, datetime.datetime(2100, 1, 1, 0, 0))
# John Smith
uid = uids[1]
self.assertEqual(uid.uuid, 'a9b403e150dd4af8953a52a4bb841051e4b705d9')
ids = self.sort_identities(uid.identities)
self.assertEqual(len(ids), 2)
prf = uid.profile
self.assertEqual(prf.uuid, 'a9b403e150dd4af8953a52a4bb841051e4b705d9')
self.assertEqual(prf.name, None)
self.assertEqual(prf.email, '*****@*****.**')
self.assertEqual(prf.gender, 'male')
self.assertEqual(prf.gender_acc, 100)
self.assertEqual(prf.is_bot, True)
self.assertEqual(prf.country_code, None)
self.assertEqual(prf.country, None)
id0 = ids[0]
self.assertEqual(id0.id, '880b3dfcb3a08712e5831bddc3dfe81fc5d7b331')
self.assertEqual(id0.name, 'John Smith')
self.assertEqual(id0.email, '*****@*****.**')
self.assertEqual(id0.username, None)
self.assertEqual(id0.source, 'scm')
id1 = ids[1]
self.assertEqual(id1.id, 'a9b403e150dd4af8953a52a4bb841051e4b705d9')
self.assertEqual(id1.name, 'John Smith')
self.assertEqual(id1.email, '*****@*****.**')
self.assertEqual(id1.username, 'jsmith')
self.assertEqual(id1.source, 'scm')
enrollments = api.enrollments(self.db, uid.uuid)
self.assertEqual(len(enrollments), 1)
rol0 = enrollments[0]
self.assertEqual(rol0.organization.name, 'Example')
self.assertEqual(rol0.start, datetime.datetime(1900, 1, 1, 0, 0))
self.assertEqual(rol0.end, datetime.datetime(2100, 1, 1, 0, 0))
