class TestBaseCase(unittest.TestCase):
"""Defines common setup and teardown methods on enroll unit tests"""
def setUp(self):
if not hasattr(sys.stdout, 'getvalue') and not hasattr(
sys.stderr, 'getvalue'):
self.fail('This test needs to be run in buffered mode')
# Create a connection to check the contents of the registry
self.db = Database(DB_USER, DB_PASSWORD, DB_NAME, DB_HOST, DB_PORT)
# Import predefined dataset for testing
self._load_test_dataset()
# Create command
self.kwargs = {
'user': DB_USER,
'password': DB_PASSWORD,
'database': DB_NAME,
'host': DB_HOST,
'port': DB_PORT
}
self.cmd = Enroll(**self.kwargs)
def tearDown(self):
self.db.clear()
def _load_test_dataset(self):
api.add_unique_identity(self.db, 'John Smith')
api.add_unique_identity(self.db, 'John Doe')
api.add_organization(self.db, 'Example')
api.add_organization(self.db, 'Bitergia')
class TestBaseCase(unittest.TestCase):
"""Defines common setup and teardown methods on countries unit tests"""
def setUp(self):
if not hasattr(sys.stdout, 'getvalue') and not hasattr(
sys.stderr, 'getvalue'):
self.fail('This test needs to be run in buffered mode')
self.db = Database(DB_USER, DB_PASSWORD, DB_NAME, DB_HOST, DB_PORT)
self._load_test_dataset()
# Create command
self.kwargs = {
'user': DB_USER,
'password': DB_PASSWORD,
'database': DB_NAME,
'host': DB_HOST,
'port': DB_PORT
}
self.cmd = Countries(**self.kwargs)
def tearDown(self):
self.db.clear()
def _load_test_dataset(self):
with self.db.connect() as session:
us = Country(code='US',
name='United States of America',
alpha3='USA')
es = Country(code='ES', name='Spain', alpha3='ESP')
gb = Country(code='GB', name='United Kingdom', alpha3='GBR')
session.add(es)
session.add(us)
session.add(gb)
class TestBaseCase(unittest.TestCase):
"""Defines common setup and teardown methods on countries unit tests"""
def setUp(self):
if not hasattr(sys.stdout, 'getvalue') and not hasattr(sys.stderr, 'getvalue'):
self.fail('This test needs to be run in buffered mode')
self.db = Database(DB_USER, DB_PASSWORD, DB_NAME, DB_HOST, DB_PORT)
self._load_test_dataset()
# Create command
self.kwargs = {'user' : DB_USER,
'password' : DB_PASSWORD,
'database' : DB_NAME,
'host' : DB_HOST,
'port' : DB_PORT}
self.cmd = Countries(**self.kwargs)
def tearDown(self):
self.db.clear()
def _load_test_dataset(self):
with self.db.connect() as session:
us = Country(code='US', name='United States of America', alpha3='USA')
es = Country(code='ES', name='Spain', alpha3='ESP')
gb = Country(code='GB', name='United Kingdom', alpha3='GBR')
session.add(es)
session.add(us)
session.add(gb)
class TestBaseCase(unittest.TestCase):
"""Defines common setup and teardown methods on enroll unit tests"""
def setUp(self):
if not hasattr(sys.stdout, 'getvalue') and not hasattr(sys.stderr, 'getvalue'):
self.fail('This test needs to be run in buffered mode')
# Create a connection to check the contents of the registry
self.db = Database(DB_USER, DB_PASSWORD, DB_NAME, DB_HOST, DB_PORT)
# Import predefined dataset for testing
self._load_test_dataset()
# Create command
self.kwargs = {'user' : DB_USER,
'password' : DB_PASSWORD,
'database' :DB_NAME,
'host' : DB_HOST,
'port' : DB_PORT}
self.cmd = Enroll(**self.kwargs)
def tearDown(self):
self.db.clear()
def _load_test_dataset(self):
api.add_unique_identity(self.db, 'John Smith')
api.add_unique_identity(self.db, 'John Doe')
api.add_organization(self.db, 'Example')
api.add_organization(self.db, 'Bitergia')
class TestBaseCase(unittest.TestCase):
"""Defines common setup and teardown methods on add unit tests"""
def setUp(self):
if not hasattr(sys.stdout, 'getvalue') and not hasattr(sys.stderr, 'getvalue'):
self.fail('This test needs to be run in buffered mode')
# Create a connection to check the contents of the registry
self.db = Database(DB_USER, DB_PASSWORD, DB_NAME, DB_HOST, DB_PORT)
self.db.clear()
self._load_test_dataset()
# Create command
self.kwargs = {'user' : DB_USER,
'password' : DB_PASSWORD,
'database' :DB_NAME,
'host' : DB_HOST,
'port' : DB_PORT}
self.cmd = Remove(**self.kwargs)
def tearDown(self):
self.db.clear()
def _load_test_dataset(self):
api.add_unique_identity(self.db, 'FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF')
api.add_identity(self.db, 'scm', '*****@*****.**',
'John Smith', uuid='FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF')
api.add_identity(self.db, 'scm', '*****@*****.**',
'John Smith', 'jsmith', uuid='FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF')
api.add_identity(self.db, 'scm', '*****@*****.**',
'John Doe', 'jdoe')
def setUp(self):
if not hasattr(sys.stdout, 'getvalue'):
self.fail('This test needs to be run in buffered mode')
# Create a dataset to test the registry
self.db = Database(DB_USER, DB_PASSWORD, DB_NAME, DB_HOST, DB_PORT)
api.add_organization(self.db, 'Example')
api.add_domain(self.db, 'Example', 'example.com')
api.add_domain(self.db, 'Example', 'example.org')
api.add_domain(self.db, 'Example', 'example.net')
api.add_organization(self.db, 'Bitergia')
api.add_domain(self.db, 'Bitergia', 'bitergia.net')
api.add_domain(self.db, 'Bitergia', 'bitergia.com', is_top_domain=True)
api.add_organization(self.db, 'LibreSoft')
# Create command
self.kwargs = {'user' : DB_USER,
'password' : DB_PASSWORD,
'database' :DB_NAME,
'host' : DB_HOST,
'port' : DB_PORT}
self.cmd = Organizations(**self.kwargs)
class TestBaseCase(unittest.TestCase):
"""Defines common setup and teardown methods on profile unit tests"""
def setUp(self):
if not hasattr(sys.stdout, 'getvalue'):
self.fail('This test needs to be run in buffered mode')
# Create a connection to check the contents of the registry
self.db = Database(DB_USER, DB_PASSWORD, DB_NAME, DB_HOST, DB_PORT)
# Import predefined dataset for testing
self._load_test_dataset()
# Create command
self.kwargs = {'user' : DB_USER,
'password' : DB_PASSWORD,
'database' :DB_NAME,
'host' : DB_HOST,
'port' : DB_PORT}
self.cmd = Profile(**self.kwargs)
def tearDown(self):
self.db.clear()
def _load_test_dataset(self):
# Add country
with self.db.connect() as session:
# Add a country
us = Country(code='US', name='United States of America', alpha3='USA')
session.add(us)
# Add identity
api.add_identity(self.db, 'scm', '*****@*****.**',
'Jane Roe', 'jroe')
class TestBlacklist(unittest.TestCase):
def setUp(self):
if not hasattr(sys.stdout, 'getvalue'):
self.fail('This test needs to be run in buffered mode')
self.db = Database(DB_USER, DB_PASSWORD, DB_NAME, DB_HOST, DB_PORT)
api.add_to_matching_blacklist(self.db, '*****@*****.**')
api.add_to_matching_blacklist(self.db, 'John Smith')
api.add_to_matching_blacklist(self.db, 'Bitergia')
api.add_to_matching_blacklist(self.db, 'John Doe')
# Create command
self.kwargs = {'user' : DB_USER,
'password' : DB_PASSWORD,
'database' :DB_NAME,
'host' : DB_HOST,
'port' : DB_PORT}
self.cmd = Blacklist(**self.kwargs)
def tearDown(self):
self.db.clear()
def test_blacklist(self):
"""Check blacklist output list"""
code = self.cmd.blacklist()
self.assertEqual(code, CMD_SUCCESS)
output = sys.stdout.getvalue().strip()
self.assertEqual(output, BLACKLIST_OUTPUT)
def test_blacklist_term(self):
"""Check if it returns the info about entries using a search term"""
code = self.cmd.blacklist('ohn')
self.assertEqual(code, CMD_SUCCESS)
output = sys.stdout.getvalue().strip()
self.assertEqual(output, BLACKLIST_OUTPUT_JOHN)
def test_not_found_term(self):
"""Check whether it prints an error for not existing entries"""
code = self.cmd.blacklist('*****@*****.**')
self.assertEqual(code, CODE_NOT_FOUND_ERROR)
output = sys.stderr.getvalue().strip()
self.assertEqual(output, BLACKLIST_NOT_FOUND_ERROR)
def test_empty_blacklist(self):
"""Check output when the blacklist is empty"""
# Delete the contents of the database
self.db.clear()
code = self.cmd.blacklist()
self.assertEqual(code, CMD_SUCCESS)
output = sys.stdout.getvalue().strip()
self.assertEqual(output, BLACKLIST_EMPTY_OUTPUT)
def __init__(self, conf):
super().__init__(conf)
self.sh_kwargs = {'user': self.db_user, 'password': self.db_password,
'database': self.db_sh, 'host': self.db_host,
'port': None}
self.db = Database(**self.sh_kwargs)
self.last_autorefresh = datetime.utcnow() # Last autorefresh date
class TestBaseCase(unittest.TestCase):
"""Defines common setup and teardown methods on affiliate unit tests"""
def setUp(self):
if not hasattr(sys.stdout, 'getvalue') and not hasattr(sys.stderr, 'getvalue'):
self.fail('This test needs to be run in buffered mode')
# Create a connection to check the contents of the registry
self.db = Database(DB_USER, DB_PASSWORD, DB_NAME, DB_HOST, DB_PORT)
self._load_test_dataset()
# Create command
self.kwargs = {'user' : DB_USER,
'password' : DB_PASSWORD,
'database' :DB_NAME,
'host' : DB_HOST,
'port' : DB_PORT}
self.cmd = Affiliate(**self.kwargs)
def tearDown(self):
self.db.clear()
def _load_test_dataset(self):
# Add some domains
api.add_organization(self.db, 'Example')
api.add_domain(self.db, 'Example', 'example.com', is_top_domain=True)
api.add_organization(self.db, 'Example Alt')
api.add_domain(self.db, 'Example Alt', 'u.example.com', is_top_domain=True)
api.add_domain(self.db, 'Example Alt', 'es.u.example.com')
api.add_domain(self.db, 'Example Alt', 'en.u.example.com')
api.add_organization(self.db, 'Bitergia')
api.add_domain(self.db, 'Bitergia', 'bitergia.com')
api.add_domain(self.db, 'Bitergia', 'bitergia.org')
api.add_organization(self.db, 'LibreSoft')
# Add some unique identities
jsmith_uuid = api.add_identity(self.db, 'scm', '*****@*****.**',
'John Smith', 'jsmith')
api.add_identity(self.db, 'scm', '*****@*****.**', 'John Smith',
uuid=jsmith_uuid)
api.add_identity(self.db, 'scm', '*****@*****.**', 'John Smith', 'jsmith',
uuid=jsmith_uuid)
api.add_enrollment(self.db, jsmith_uuid, 'Bitergia')
# Add John Doe identity
api.add_identity(self.db, 'unknown', None, 'John Doe', 'jdoe')
# Add Jane Rae identity
jroe_uuid = api.add_identity(self.db, 'scm', '*****@*****.**',
'Jane Roe', 'jroe')
api.add_identity(self.db, 'scm', '*****@*****.**',
uuid=jroe_uuid)
api.add_identity(self.db, 'unknown', '*****@*****.**',
uuid=jroe_uuid)
class TestBaseCase(unittest.TestCase):
"""Defines common setup and teardown methods on add unit tests"""
def setUp(self):
if not hasattr(sys.stdout, 'getvalue') and not hasattr(sys.stderr, 'getvalue'):
self.fail('This test needs to be run in buffered mode')
# Create a connection to check the contents of the registry
self.db = Database(DB_USER, DB_PASSWORD, DB_NAME, DB_HOST, DB_PORT)
self.db.clear()
self._load_test_dataset()
# Create command
self.kwargs = {'user' : DB_USER,
'password' : DB_PASSWORD,
'database' :DB_NAME,
'host' : DB_HOST,
'port' : DB_PORT}
self.cmd = Merge(**self.kwargs)
def tearDown(self):
self.db.clear()
def _load_test_dataset(self):
# Add country
with self.db.connect() as session:
# Add a country
us = Country(code='US', name='United States of America', alpha3='USA')
session.add(us)
api.add_unique_identity(self.db, 'John Smith')
api.add_identity(self.db, 'scm', '*****@*****.**',
uuid='John Smith')
api.add_identity(self.db, 'scm', '*****@*****.**', 'John Smith',
uuid='John Smith')
api.edit_profile(self.db, 'John Smith', name='John Smith', is_bot=False)
api.add_unique_identity(self.db, 'John Doe')
api.add_identity(self.db, 'scm', '*****@*****.**',
uuid='John Doe')
api.edit_profile(self.db, 'John Doe', email='*****@*****.**', is_bot=True,
country_code='US')
api.add_organization(self.db, 'Example')
api.add_enrollment(self.db, 'John Smith', 'Example')
api.add_enrollment(self.db, 'John Doe', 'Example')
api.add_organization(self.db, 'Bitergia')
api.add_enrollment(self.db, 'John Smith', 'Bitergia')
api.add_enrollment(self.db, 'John Doe', 'Bitergia',
datetime.datetime(1999, 1, 1),
datetime.datetime(2000, 1, 1))
api.add_organization(self.db, 'LibreSoft')
def setUp(self):
config = Config(CONF_FILE)
sh = config.get_conf()['sortinghat']
self.sh_kwargs = {'user': sh['user'], 'password': sh['password'],
'database': sh['database'], 'host': sh['host'],
'port': None}
# Clean the database to start an empty state
Database.drop(**self.sh_kwargs)
# Create command
Database.create(**self.sh_kwargs)
self.sh_db = Database(**self.sh_kwargs)
def setUp(self):
config = Config(CONF_FILE)
sh = config.get_conf()['sortinghat']
self.sh_kwargs = {'user': sh['user'], 'password': sh['password'],
'database': sh['database'], 'host': sh['host'],
'port': None}
# Clean the database to start an empty state
Database.drop(**self.sh_kwargs)
# Create command
Database.create(**self.sh_kwargs)
self.sh_db = Database(**self.sh_kwargs)
def __init__(self, conf, load_orgs=True, load_ids=True, unify=True,
autoprofile=True, affiliate=True, bots=True):
super().__init__(conf)
self.load_ids = load_ids # Load identities from raw index
self.unify = unify # Unify identities
self.autoprofile = autoprofile # Execute autoprofile
self.affiliate = affiliate # Affiliate identities
self.bots = bots # Mark bots in SH
self.sh_kwargs = {'user': self.db_user, 'password': self.db_password,
'database': self.db_sh, 'host': self.db_host,
'port': None}
self.db = Database(**self.sh_kwargs)
self.last_autorefresh = datetime.utcnow() # Last autorefresh date
def setUp(self):
if not hasattr(sys.stdout, 'getvalue'):
self.fail('This test needs to be run in buffered mode')
# Create a connection to check the contents of the registry
self.db = Database(DB_USER, DB_PASSWORD, DB_NAME, DB_HOST, DB_PORT)
# Create command
self.kwargs = {'user' : DB_USER,
'password' : DB_PASSWORD,
'database' :DB_NAME,
'host' : DB_HOST,
'port' : DB_PORT}
self.cmd = Organizations(**self.kwargs)
def setUpClass(cls):
config = configparser.ConfigParser()
config.read(CONFIG_FILE)
cls.db_kwargs = {'user': config['Database']['user'],
'password': config['Database']['password'],
'database': config['Database']['name'],
'host': config['Database']['host'],
'port': config['Database']['port']}
if 'create' in config['Database']:
cls.create = config['Database'].getboolean('create')
else:
cls.create = False
if cls.create:
Database.create(**cls.db_kwargs)
cls.db = Database(**cls.db_kwargs)
cls.db.clear()
def __init__(self,
db_sortinghat=None,
db_projects_map=None,
insecure=True):
self.sortinghat = False
if db_sortinghat and not SORTINGHAT_LIBS:
raise RuntimeError(
"Sorting hat configured but libraries not available.")
if db_sortinghat:
self.sh_db = Database("root", "", db_sortinghat, "mariadb")
self.sortinghat = True
self.prjs_map = None
if db_projects_map and not MYSQL_LIBS:
raise RuntimeError(
"Projects configured but MySQL libraries not available.")
if db_projects_map:
self.prjs_map = self._get_projects_map(db_projects_map)
self.requests = requests.Session()
if insecure:
requests.packages.urllib3.disable_warnings(
requests.packages.urllib3.exceptions.InsecureRequestWarning)
self.requests.verify = False
def __init__(self, conf):
super().__init__(conf)
self.sh_kwargs = {'user': self.db_user, 'password': self.db_password,
'database': self.db_sh, 'host': self.db_host,
'port': None}
self.db = Database(**self.sh_kwargs)
self.last_autorefresh = datetime.utcnow() # Last autorefresh date
class TestBaseCase(unittest.TestCase):
"""Defines common setup and teardown methods on unify unit tests"""
def setUp(self):
if not hasattr(sys.stdout, 'getvalue') and not hasattr(sys.stderr, 'getvalue'):
self.fail('This test needs to be run in buffered mode')
# Create a connection to check the contents of the registry
self.db = Database(DB_USER, DB_PASSWORD, DB_NAME, DB_HOST, DB_PORT)
self._load_test_dataset()
# Create command
self.kwargs = {'user' : DB_USER,
'password' : DB_PASSWORD,
'database' :DB_NAME,
'host' : DB_HOST,
'port' : DB_PORT}
self.cmd = Unify(**self.kwargs)
def tearDown(self):
self.db.clear()
def _load_test_dataset(self):
# Add some unique identities
uuid = api.add_identity(self.db, source='scm', email='*****@*****.**', name='John Smith')
api.add_identity(self.db, source='scm', name='John Smith', uuid=uuid)
api.add_identity(self.db, source='scm', username='******', uuid=uuid)
uuid = api.add_identity(self.db, source='alt', name='J. Smith', username='******')
api.add_identity(self.db, source='alt', name='John Smith', username='******', uuid=uuid)
api.add_identity(self.db, source='alt', email='jsmith', uuid=uuid)
uuid = api.add_identity(self.db, source='scm', name='Jane Rae')
api.add_identity(self.db, source='mls', email='*****@*****.**', name='Jane Rae Doe', uuid=uuid)
uuid = api.add_identity(self.db, source='scm', name='J. Smith', username='******')
api.add_identity(self.db, source='scm', username='******', uuid=uuid)
api.add_identity(self.db, source='mls', username='******', uuid=uuid)
api.add_identity(self.db, source='mls', email='*****@*****.**', name='Smith. J', uuid=uuid)
uuid = api.add_identity(self.db, source='mls', email='*****@*****.**', name='Jane Rae Doe')
api.add_identity(self.db, source='mls', name='jrae', uuid=uuid)
uuid = api.add_identity(self.db, source='scm', name='jrae')
def setUp(self):
if not hasattr(sys.stdout, 'getvalue') and not hasattr(
sys.stderr, 'getvalue'):
self.fail('This test needs to be run in buffered mode')
self.db = Database(DB_USER, DB_PASSWORD, DB_NAME, DB_HOST, DB_PORT)
self._load_test_dataset()
# Create command
self.kwargs = {
'user': DB_USER,
'password': DB_PASSWORD,
'database': DB_NAME,
'host': DB_HOST,
'port': DB_PORT
}
self.cmd = Countries(**self.kwargs)
def setUpClass(cls):
config = configparser.ConfigParser()
config.read(CONFIG_FILE)
cls.db_kwargs = {
'user': config['Database']['user'],
'password': config['Database']['password'],
'database': config['Database']['name'],
'host': config['Database']['host'],
'port': config['Database']['port']
}
if 'create' in config['Database']:
cls.create = config['Database'].getboolean('create')
else:
cls.create = False
if cls.create:
Database.create(**cls.db_kwargs)
cls.db = Database(**cls.db_kwargs)
cls.db.clear()
class TestBaseCase(unittest.TestCase):
"""Defines common setup and teardown methods on show unit tests"""
def setUp(self):
if not hasattr(sys.stdout, 'getvalue'):
self.fail('This test needs to be run in buffered mode')
# Create a connection to check the contents of the registry
self.db = Database(DB_USER, DB_PASSWORD, DB_NAME, DB_HOST, DB_PORT)
self._load_test_dataset()
# Create command
self.kwargs = {
'user': DB_USER,
'password': DB_PASSWORD,
'database': DB_NAME,
'host': DB_HOST,
'port': DB_PORT
}
self.cmd = Load(**self.kwargs)
def _load_test_dataset(self):
# Add country
with self.db.connect() as session:
# Add a country
us = Country(code='US',
name='United States of America',
alpha3='USA')
session.add(us)
def tearDown(self):
self.db.clear()
def get_parser(self, filename):
if sys.version_info[0] >= 3: # Python 3
with open(filename, 'r', encoding='UTF-8') as f:
content = f.read()
else: # Python 2
with open(filename, 'r') as f:
content = f.read().decode('UTF-8')
return SortingHatParser(content)
def sort_identities(self, ids):
return sorted(ids, key=lambda x: x.id)
def setUpClass(cls):
config = configparser.ConfigParser()
config.read(CONFIG_FILE)
cls.db_kwargs = {'user': config['Database']['user'],
'password': config['Database']['password'],
'database': config['Database']['name'],
'host': config['Database']['host'],
'port': config['Database']['port']}
cls.db = Database(**cls.db_kwargs)
cls.db.clear()
def sortinghat_db_conn():
"""
Returns Sorting Hat database object to work with
"""
sortinghat_db = Database(user=Conf.shdb_user,
password=Conf.shdb_pass,
database=Conf.shdb_name,
host=Conf.shdb_host)
return sortinghat_db
def __init__(self, db_sortinghat=None, db_projects_map=None, json_projects_map=None,
db_user='', db_password='', db_host='', insecure=True):
perceval_backend = None
super().__init__(perceval_backend, insecure=insecure)
self.sortinghat = False
if db_user == '':
db_user = DEFAULT_DB_USER
if db_sortinghat and not SORTINGHAT_LIBS:
raise RuntimeError("Sorting hat configured but libraries not available.")
if db_sortinghat:
# self.sh_db = Database("root", "", db_sortinghat, "mariadb")
if not Enrich.sh_db:
Enrich.sh_db = Database(db_user, db_password, db_sortinghat, db_host)
self.sortinghat = True
self.prjs_map = None # mapping beetween repositories and projects
self.json_projects = None
if json_projects_map:
with open(json_projects_map) as data_file:
self.json_projects = json.load(data_file)
# If we have JSON projects always use them for mapping
self.prjs_map = self.__convert_json_to_projects_map(self.json_projects)
if not self.json_projects:
if db_projects_map and not MYSQL_LIBS:
raise RuntimeError("Projects configured but MySQL libraries not available.")
if db_projects_map and not self.json_projects:
self.prjs_map = self.__get_projects_map(db_projects_map,
db_user, db_password,
db_host)
if self.prjs_map and self.json_projects:
# logger.info("Comparing db and json projects")
# self.__compare_projects_map(self.prjs_map, self.json_projects)
pass
self.studies = []
self.requests = grimoire_con()
self.elastic = None
self.type_name = "items" # type inside the index to store items enriched
# To add the gelk version to enriched items
self.gelk_version = __version__
# params used to configure the backend
# in perceval backends managed directly inside the backend
# in twitter and others managed in arthur logic
self.backend_params = None
# Label used during enrichment for identities without a known affiliation
self.unaffiliated_group = 'Unknown'
# Label used during enrichment for identities with no gender info
self.unknown_gender = 'Unknown'
def sortinghat_db_conn():
"""
Returns Sorting Hat database object to work with
"""
# shdb_user, shdb_pass, shdb_name, shdb_host = parse_shdb_config_file(filename)
sortinghat_db = Database(user=Conf.shdb_user,
password=Conf.shdb_pass,
database=Conf.shdb_name,
host=Conf.shdb_host)
return sortinghat_db
def setUp(self):
if not hasattr(sys.stdout, 'getvalue') and not hasattr(
sys.stderr, 'getvalue'):
self.fail('This test needs to be run in buffered mode')
# Create a connection to check the contents of the registry
self.db = Database(DB_USER, DB_PASSWORD, DB_NAME, DB_HOST, DB_PORT)
# Import predefined dataset for testing
self._load_test_dataset()
# Create command
self.kwargs = {
'user': DB_USER,
'password': DB_PASSWORD,
'database': DB_NAME,
'host': DB_HOST,
'port': DB_PORT
}
self.cmd = Enroll(**self.kwargs)
class TestBaseCase(unittest.TestCase):
"""Defines common setup and teardown methods on show unit tests"""
def setUp(self):
if not hasattr(sys.stdout, 'getvalue'):
self.fail('This test needs to be run in buffered mode')
# Create a connection to check the contents of the registry
self.db = Database(DB_USER, DB_PASSWORD, DB_NAME, DB_HOST, DB_PORT)
self._load_test_dataset()
# Create command
self.kwargs = {'user' : DB_USER,
'password' : DB_PASSWORD,
'database' : DB_NAME,
'host' : DB_HOST,
'port' : DB_PORT}
self.cmd = Load(**self.kwargs)
def _load_test_dataset(self):
# Add country
with self.db.connect() as session:
# Add a country
us = Country(code='US', name='United States of America', alpha3='USA')
session.add(us)
def tearDown(self):
self.db.clear()
def get_parser(self, filename):
if sys.version_info[0] >= 3: # Python 3
with open(filename, 'r', encoding='UTF-8') as f:
content = f.read()
else: # Python 2
with open(filename, 'r') as f:
content = f.read().decode('UTF-8')
return SortingHatParser(content)
def sort_identities(self, ids):
return sorted(ids, key=lambda x: x.id)
def __init__(
self,
db_projects_map=None,
db_sortinghat=None,
):
self.sortinghat = False
if db_sortinghat:
self.sh_db = Database("root", "", db_sortinghat, "mariadb")
self.sortinghat = True
self.prjs_map = None
if db_projects_map:
self.prjs_map = self._get_projects_map(db_projects_map)
def test_init(self):
"""Check registry initialization"""
code = self.cmd.run(self.name)
self.assertEqual(code, CMD_SUCCESS)
db = Database(database=self.name, **self.kwargs)
self.assertIsInstance(db, Database)
# Check if the list of countries was loaded
countries = api.countries(db)
self.assertEqual(len(countries), 249)
def test_initialize(self):
"""Check registry initialization"""
code = self.cmd.initialize(self.name)
self.assertEqual(code, CMD_SUCCESS)
db = Database(self.kwargs['user'], self.kwargs['password'], self.name,
self.kwargs['host'], self.kwargs['port'])
self.assertIsInstance(db, Database)
# Check if the list of countries was loaded
countries = api.countries(db)
self.assertEqual(len(countries), 249)
def __init__(self, db_sortinghat=None, db_projects_map=None, insecure=True):
self.sortinghat = False
if db_sortinghat:
self.sh_db = Database("root", "", db_sortinghat, "mariadb")
self.sortinghat = True
self.prjs_map = None
if db_projects_map:
self.prjs_map = self._get_projects_map(db_projects_map)
self.requests = requests.Session()
if insecure:
requests.packages.urllib3.disable_warnings(requests.packages.urllib3.exceptions.InsecureRequestWarning)
self.requests.verify = False
def __init__(self, config, backend_section=None):
super().__init__(config)
self.backend_section = backend_section
# This will be options in next iteration
self.clean = False
# check whether the aliases has beed already created
self.enrich_aliases = False
self.sh_kwargs = {'user': self.db_user, 'password': self.db_password,
'database': self.db_sh, 'host': self.db_host,
'port': None}
self.db = Database(**self.sh_kwargs)
autorefresh_interval = self.conf['es_enrichment']['autorefresh_interval']
self.last_autorefresh = self.__update_last_autorefresh(days=autorefresh_interval)
self.last_autorefresh_studies = self.last_autorefresh
class TestBaseCase(unittest.TestCase):
"""Defines common setup and teardown methods on profile unit tests"""
def setUp(self):
if not hasattr(sys.stdout, 'getvalue'):
self.fail('This test needs to be run in buffered mode')
# Create a connection to check the contents of the registry
self.db = Database(DB_USER, DB_PASSWORD, DB_NAME, DB_HOST, DB_PORT)
# Import predefined dataset for testing
self._load_test_dataset()
# Create command
self.kwargs = {
'user': DB_USER,
'password': DB_PASSWORD,
'database': DB_NAME,
'host': DB_HOST,
'port': DB_PORT
}
self.cmd = Profile(**self.kwargs)
def tearDown(self):
self.db.clear()
def _load_test_dataset(self):
# Add country
with self.db.connect() as session:
# Add a country
us = Country(code='US',
name='United States of America',
alpha3='USA')
session.add(us)
# Add identity
api.add_identity(self.db, 'scm', '*****@*****.**', 'Jane Roe',
'jroe')
def setUp(self):
if not hasattr(sys.stdout, 'getvalue') and not hasattr(sys.stderr, 'getvalue'):
self.fail('This test needs to be run in buffered mode')
self.db = Database(DB_USER, DB_PASSWORD, DB_NAME, DB_HOST, DB_PORT)
self._load_test_dataset()
# Create command
self.kwargs = {'user' : DB_USER,
'password' : DB_PASSWORD,
'database' : DB_NAME,
'host' : DB_HOST,
'port' : DB_PORT}
self.cmd = Countries(**self.kwargs)
def __init__(self, config, backend_section=None):
super().__init__(config)
self.backend_section = backend_section
# This will be options in next iteration
self.clean = False
# check whether the aliases has beed already created
self.enrich_aliases = False
self.sh_kwargs = {
'user': self.db_user,
'password': self.db_password,
'database': self.db_sh,
'host': self.db_host,
'port': None
}
self.db = Database(**self.sh_kwargs)
self.last_autorefresh = datetime.utcnow() # Last autorefresh date
def __init__(self, config, backend_section=None):
super().__init__(config)
self.backend_section = backend_section
# This will be options in next iteration
self.clean = False
# check whether the aliases has beed already created
self.enrich_aliases = False
if self.db_sh is None and self.db_host is None:
self.db = None
else:
self.db = Database(**self.sh_kwargs)
autorefresh_interval = self.conf['es_enrichment']['autorefresh_interval']
self.last_autorefresh = self.__update_last_autorefresh(days=autorefresh_interval)
self.last_autorefresh_studies = self.last_autorefresh
self.last_sortinghat_import = None
def setUp(self):
if not hasattr(sys.stdout, 'getvalue') and not hasattr(sys.stderr, 'getvalue'):
self.fail('This test needs to be run in buffered mode')
# Create a connection to check the contents of the registry
self.db = Database(DB_USER, DB_PASSWORD, DB_NAME, DB_HOST, DB_PORT)
# Import predefined dataset for testing
self._load_test_dataset()
# Create command
self.kwargs = {'user' : DB_USER,
'password' : DB_PASSWORD,
'database' :DB_NAME,
'host' : DB_HOST,
'port' : DB_PORT}
self.cmd = Enroll(**self.kwargs)
def setUp(self):
if not hasattr(sys.stdout, 'getvalue'):
self.fail('This test needs to be run in buffered mode')
self.db = Database(DB_USER, DB_PASSWORD, DB_NAME, DB_HOST, DB_PORT)
api.add_to_matching_blacklist(self.db, '*****@*****.**')
api.add_to_matching_blacklist(self.db, 'John Smith')
api.add_to_matching_blacklist(self.db, 'Bitergia')
api.add_to_matching_blacklist(self.db, 'John Doe')
# Create command
self.kwargs = {'user' : DB_USER,
'password' : DB_PASSWORD,
'database' :DB_NAME,
'host' : DB_HOST,
'port' : DB_PORT}
self.cmd = Blacklist(**self.kwargs)
def main():
"""Link Sorting Hat unique identities to Metrics Grimoire identities"""
args = parse_args()
conn = open_database(args, args.database)
people_uidentities = retrieve_people_uidentities(conn.cursor())
people_upeople = retrieve_people_upeople(conn.cursor())
close_database(conn)
conn = open_database(args, args.identities)
enrollments = retrieve_upeople_companies(conn.cursor())
close_database(conn)
identities = find_matches(people_uidentities, people_upeople, enrollments)
db = Database(args.user, args.password, args.sortinghat, args.host,
args.port)
merge_identities(db, identities)
enroll_identities(db, identities, args.insert_orgs)
class TestOrgsRegistry(unittest.TestCase):
def setUp(self):
if not hasattr(sys.stdout, 'getvalue'):
self.fail('This test needs to be run in buffered mode')
# Create a dataset to test the registry
self.db = Database(DB_USER, DB_PASSWORD, DB_NAME, DB_HOST, DB_PORT)
api.add_organization(self.db, 'Example')
api.add_domain(self.db, 'Example', 'example.com')
api.add_domain(self.db, 'Example', 'example.org')
api.add_domain(self.db, 'Example', 'example.net')
api.add_organization(self.db, 'Bitergia')
api.add_domain(self.db, 'Bitergia', 'bitergia.net')
api.add_domain(self.db, 'Bitergia', 'bitergia.com', is_top_domain=True)
api.add_organization(self.db, 'LibreSoft')
# Create command
self.kwargs = {'user' : DB_USER,
'password' : DB_PASSWORD,
'database' :DB_NAME,
'host' : DB_HOST,
'port' : DB_PORT}
self.cmd = Organizations(**self.kwargs)
def tearDown(self):
self.db.clear()
def test_registry(self):
"""Check registry output list"""
code = self.cmd.registry()
self.assertEqual(code, CMD_SUCCESS)
output = sys.stdout.getvalue().strip()
self.assertEqual(output, REGISTRY_OUTPUT)
def test_registry_term(self):
"""Check if it returns the info about orgs using a search term"""
# Add an extra organization first
api.add_organization(self.db, 'MyExample')
api.add_domain(self.db, 'MyExample', 'myexample.com')
code = self.cmd.registry('Example')
self.assertEqual(code, CMD_SUCCESS)
output = sys.stdout.getvalue().strip()
self.assertEqual(output, REGISTRY_OUTPUT_EXAMPLE)
def test_not_found_term(self):
"""Check whether it prints an error for not existing organizations"""
code = self.cmd.registry('Bitergium')
self.assertEqual(code, CODE_NOT_FOUND_ERROR)
output = sys.stderr.getvalue().strip()
self.assertEqual(output, REGISTRY_ORG_NOT_FOUND_ERROR)
def test_empty_registry(self):
"""Check output when the registry is empty"""
# Delete the contents of the database
self.db.clear()
code = self.cmd.registry()
self.assertEqual(code, CMD_SUCCESS)
output = sys.stdout.getvalue().strip()
self.assertEqual(output, REGISTRY_EMPTY_OUTPUT)
class TestOrgsDelete(unittest.TestCase):
def setUp(self):
if not hasattr(sys.stdout, 'getvalue'):
self.fail('This test needs to be run in buffered mode')
# Create a connection to check the contents of the registry
self.db = Database(DB_USER, DB_PASSWORD, DB_NAME, DB_HOST, DB_PORT)
# Create command
self.kwargs = {'user' : DB_USER,
'password' : DB_PASSWORD,
'database' :DB_NAME,
'host' : DB_HOST,
'port' : DB_PORT}
self.cmd = Organizations(**self.kwargs)
def tearDown(self):
self.db.clear()
def test_delete(self):
"""Check whether everything works ok when deleting organizations and domains"""
# First, add a set of organizations, including some domains
self.cmd.add('Example')
self.cmd.add('Example', 'example.com')
self.cmd.add('Example', 'example.org')
self.cmd.add('Example', 'example.net')
self.cmd.add('Bitergia')
self.cmd.add('Bitergia', 'bitergia.com')
self.cmd.add('LibreSoft')
self.cmd.add('Bitergium')
self.cmd.add('Bitergium', 'bitergium.com')
self.cmd.add('Bitergium', 'bitergium.net')
# Delete an organization
orgs = api.registry(self.db, 'Bitergia')
self.assertEqual(len(orgs), 1)
code = self.cmd.delete('Bitergia')
self.assertEqual(code, CMD_SUCCESS)
self.assertRaises(NotFoundError, api.registry,
self.db, 'Bitergia')
# Delete a domain
orgs = api.registry(self.db, 'Bitergium')
self.assertEqual(len(orgs[0].domains), 2)
code = self.cmd.delete('Bitergium', 'bitergium.com')
self.assertEqual(code, CMD_SUCCESS)
orgs = api.registry(self.db, 'Bitergium')
self.assertEqual(len(orgs[0].domains), 1)
# Delete organization with several domains
orgs = api.registry(self.db, 'Example')
self.assertEqual(len(orgs), 1)
code = self.cmd.delete('Example')
self.assertEqual(code, CMD_SUCCESS)
self.assertRaises(NotFoundError, api.registry,
self.db, 'Example')
# The final content of the registry should have
# two companies and one domain
orgs = api.registry(self.db)
self.assertEqual(len(orgs), 2)
org1 = orgs[0]
self.assertEqual(org1.name, 'Bitergium')
doms1 = org1.domains
self.assertEqual(len(doms1), 1)
self.assertEqual(doms1[0].domain, 'bitergium.net')
org2 = orgs[1]
self.assertEqual(org2.name, 'LibreSoft')
doms2 = org2.domains
self.assertEqual(len(doms2), 0)
def test_not_found_organization(self):
"""Check if it fails removing an organization that does not exists"""
# It should print an error when the registry is empty
code = self.cmd.delete('Bitergium')
self.assertEqual(code, CODE_NOT_FOUND_ERROR)
output = sys.stderr.getvalue().strip().split('\n')[0]
self.assertEqual(output, REGISTRY_ORG_NOT_FOUND_ERROR)
# Add a pair of organizations to check delete with a registry
# with contents
self.cmd.add('Example')
self.cmd.add('Bitergia')
self.cmd.add('Bitergia', 'bitergia.com')
# The error should be the same
code = self.cmd.delete('Bitergium')
self.assertEqual(code, CODE_NOT_FOUND_ERROR)
output = sys.stderr.getvalue().strip().split('\n')[1]
self.assertEqual(output, REGISTRY_ORG_NOT_FOUND_ERROR)
# It fails again, when trying to delete a domain from
# a organization that does not exist
code = self.cmd.delete('LibreSoft', 'bitergium.com')
self.assertEqual(code, CODE_NOT_FOUND_ERROR)
output = sys.stderr.getvalue().strip().split('\n')[2]
self.assertEqual(output, REGISTRY_ORG_NOT_FOUND_ERROR_ALT)
# Nothing has been deleted from the registry
orgs = api.registry(self.db)
self.assertEqual(len(orgs), 2)
self.assertEqual(len(orgs[0].domains), 1)
self.assertEqual(len(orgs[1].domains), 0)
def test_not_found_domain(self):
"""Check if it fails removing an domain that does not exists"""
# Add a pair of organizations to check delete with a registry
# with contents
self.cmd.add('Example')
self.cmd.add('Bitergia')
self.cmd.add('Bitergia', 'bitergia.com')
code = self.cmd.delete('Example', 'example.com')
self.assertEqual(code, CODE_NOT_FOUND_ERROR)
output = sys.stderr.getvalue().strip().split('\n')[0]
self.assertEqual(output, REGISTRY_DOM_NOT_FOUND_ERROR)
# It should not fail because the domain is assigned
# to other organization
code = self.cmd.delete('Example', 'bitergia.com')
self.assertEqual(code, CODE_NOT_FOUND_ERROR)
output = sys.stderr.getvalue().strip().split('\n')[1]
self.assertEqual(output, REGISTRY_DOM_NOT_FOUND_ERROR_ALT)
# Nothing has been deleted from the registry
orgs = api.registry(self.db)
self.assertEqual(len(orgs), 2)
self.assertEqual(len(orgs[0].domains), 1)
self.assertEqual(len(orgs[1].domains), 0)
class TestOrgsAdd(unittest.TestCase):
def setUp(self):
if not hasattr(sys.stdout, 'getvalue'):
self.fail('This test needs to be run in buffered mode')
# Create a connection to check the contents of the registry
self.db = Database(DB_USER, DB_PASSWORD, DB_NAME, DB_HOST, DB_PORT)
# Create command
self.kwargs = {'user' : DB_USER,
'password' : DB_PASSWORD,
'database' :DB_NAME,
'host' : DB_HOST,
'port' : DB_PORT}
self.cmd = Organizations(**self.kwargs)
def tearDown(self):
self.db.clear()
def test_add(self):
"""Check whether everything works ok when adding organizations and domains"""
code = self.cmd.add('Example')
self.assertEqual(code, CMD_SUCCESS)
code = self.cmd.add('Example', 'example.com')
self.assertEqual(code, CMD_SUCCESS)
code = self.cmd.add('Bitergia')
self.assertEqual(code, CMD_SUCCESS)
code = self.cmd.add('Bitergia', 'bitergia.net')
self.assertEqual(code, CMD_SUCCESS)
code = self.cmd.add('Bitergia', 'bitergia.com', is_top_domain=True)
self.assertEqual(code, CMD_SUCCESS)
code = self.cmd.add('LibreSoft', '') # This will work like adding a organization
self.assertEqual(code, CMD_SUCCESS)
code = self.cmd.add('Example', 'example.org')
self.assertEqual(code, CMD_SUCCESS)
code = self.cmd.add('Example', 'example.net')
self.assertEqual(code, CMD_SUCCESS)
# List the registry and check the output
self.cmd.registry()
output = sys.stdout.getvalue().strip()
self.assertEqual(output, REGISTRY_OUTPUT)
def test_existing_organization(self):
"""Check if it fails adding an organization that already exists"""
code1 = self.cmd.add('Bitergium')
self.assertEqual(code1, CMD_SUCCESS)
code2 = self.cmd.add('Bitergium')
self.assertEqual(code2, CODE_ALREADY_EXISTS_ERROR)
output = sys.stderr.getvalue().strip()
self.assertEqual(output, REGISTRY_ORG_ALREADY_EXISTS_ERROR)
def test_non_existing_organization(self):
"""Check if it fails adding domains to not existing organizations"""
code = self.cmd.add('Bitergium', 'bitergium.com')
self.assertEqual(code, CODE_NOT_FOUND_ERROR)
output = sys.stderr.getvalue().strip()
self.assertEqual(output, REGISTRY_ORG_NOT_FOUND_ERROR)
def test_existing_domain(self):
"""Check if it fails adding a domain that already exists"""
# Add a pair of organizations and domains first
self.cmd.add('Example')
self.cmd.add('Example', 'example.com')
self.cmd.add('Bitergia')
self.cmd.add('Bitergia', 'bitergia.com')
# Add 'bitergia.com' to 'Example' org
# It should print an error
code = self.cmd.add('Example', 'bitergia.com')
self.assertEqual(code, CODE_ALREADY_EXISTS_ERROR)
output = sys.stderr.getvalue().strip()
self.assertEqual(output, REGISTRY_DOM_ALREADY_EXISTS_ERROR)
def test_overwrite_domain(self):
"""Check whether it overwrites the old organization-domain relationship
and the top_domain flag"""
# Add a pair of organizations and domains first
self.cmd.add('Example')
self.cmd.add('Example', 'example.com')
self.cmd.add('Example', 'example.org')
self.cmd.add('Bitergia')
self.cmd.add('Bitergia', 'bitergia.com')
# Overwrite the relationship assigning the domain to a different
# company and top_domain flag
code = self.cmd.add('Bitergia', 'example.com',
is_top_domain=True, overwrite=True)
self.assertEqual(code, CMD_SUCCESS)
output = sys.stdout.getvalue().strip()
self.assertEqual(output, REGISTRY_EMPTY_OUTPUT)
# Check if the domain has been assigned to Bitergia
orgs = api.registry(self.db)
org1 = orgs[0]
self.assertEqual(org1.name, 'Bitergia')
doms1 = org1.domains
doms1.sort(key=lambda x: x.domain)
self.assertEqual(len(doms1), 2)
dom = doms1[0]
self.assertEqual(dom.domain, 'bitergia.com')
dom = doms1[1]
self.assertEqual(dom.domain, 'example.com')
self.assertEqual(dom.is_top_domain, True)
org2 = orgs[1]
self.assertEqual(org2.name, 'Example')
doms2 = org2.domains
doms2.sort(key=lambda x: x.domain)
self.assertEqual(len(doms2), 1)
dom1 = doms2[0]
self.assertEqual(dom1.domain, 'example.org')
def test_none_organization(self):
"""Check behavior adding None organizations"""
code = self.cmd.add(None)
self.assertEqual(code, CMD_SUCCESS)
output = sys.stdout.getvalue().strip()
self.assertEqual(output, REGISTRY_EMPTY_OUTPUT)
# The registry should be empty
orgs = api.registry(self.db)
self.assertEqual(len(orgs), 0)
def test_empty_organization(self):
"""Check behavior adding empty organizations"""
code = self.cmd.add('')
self.assertEqual(code, CMD_SUCCESS)
output = sys.stdout.getvalue().strip()
self.assertEqual(output, REGISTRY_EMPTY_OUTPUT)
# The registry should be empty
orgs = api.registry(self.db)
self.assertEqual(len(orgs), 0)
class TestBlacklistCommand(unittest.TestCase):
"""Blacklist command unit tests"""
def setUp(self):
if not hasattr(sys.stdout, 'getvalue'):
self.fail('This test needs to be run in buffered mode')
# Create a connection to check the contents of the registry
self.db = Database(DB_USER, DB_PASSWORD, DB_NAME, DB_HOST, DB_PORT)
# Create command
self.kwargs = {'user' : DB_USER,
'password' : DB_PASSWORD,
'database' :DB_NAME,
'host' : DB_HOST,
'port' : DB_PORT}
self.cmd = Blacklist(**self.kwargs)
def tearDown(self):
self.db.clear()
def test_default_action(self):
"""Check whether when no action is given it runs --list"""
self.__load_test_dataset()
code = self.cmd.run()
self.assertEqual(code, CMD_SUCCESS)
output = sys.stdout.getvalue().strip()
self.assertEqual(output, BLACKLIST_OUTPUT)
def test_list_without_args(self):
"""Test list action with and without arguments"""
self.__load_test_dataset()
code = self.cmd.run('-l')
self.assertEqual(code, CMD_SUCCESS)
output = sys.stdout.getvalue().strip()
self.assertEqual(output, BLACKLIST_OUTPUT)
def test_list_with_args(self):
"""Test list action with arguments"""
self.__load_test_dataset()
code = self.cmd.run('--list', 'ohn')
self.assertEqual(code, CMD_SUCCESS)
output = sys.stdout.getvalue().strip()
self.assertEqual(output, BLACKLIST_OUTPUT_JOHN)
def test_add_with_args(self):
"""Test add action"""
code = self.cmd.run('--add', 'John Smith')
self.assertEqual(code, CMD_SUCCESS)
code = self.cmd.run('-a', 'Bitergia')
self.assertEqual(code, CMD_SUCCESS)
code = self.cmd.run('--add', 'John Doe')
self.assertEqual(code, CMD_SUCCESS)
code = self.cmd.run('-a', '*****@*****.**')
self.assertEqual(code, CMD_SUCCESS)
code = self.cmd.run('--list')
self.assertEqual(code, CMD_SUCCESS)
# Check output
output = sys.stdout.getvalue().strip()
self.assertEqual(output, BLACKLIST_OUTPUT)
def test_add_without_args(self):
"""Check when calling --add without args, it does not do anything"""
code = self.cmd.run('--add')
self.assertEqual(code, CMD_SUCCESS)
code = self.cmd.run('-l')
self.assertEqual(code, CMD_SUCCESS)
output = sys.stdout.getvalue().strip()
self.assertEqual(output, BLACKLIST_EMPTY_OUTPUT)
def test_delete_with_args(self):
"""Test delete action"""
self.__load_test_dataset()
# Delete contents
code = self.cmd.run('--delete', 'Bitergia')
self.assertEqual(code, CMD_SUCCESS)
code = self.cmd.run('-d', 'John Doe')
self.assertEqual(code, CMD_SUCCESS)
code = self.cmd.run('--list')
self.assertEqual(code, CMD_SUCCESS)
# Check output
output = sys.stdout.getvalue().strip()
self.assertEqual(output, BLACKLIST_OUTPUT_ALT)
def test_delete_without_args(self):
"""Check when calling --delete without args, it does not do anything"""
self.__load_test_dataset()
code = self.cmd.run('--delete')
self.assertEqual(code, CMD_SUCCESS)
code = self.cmd.run('-l')
self.assertEqual(code, CMD_SUCCESS)
# Check output
output = sys.stdout.getvalue().strip()
self.assertEqual(output, BLACKLIST_OUTPUT)
def test_run_mixing_actions(self):
"""Check how it works when mixing actions"""
self.cmd.run('--add', 'John Doe')
self.cmd.run('-a', 'John Smith')
self.cmd.run('-a', 'Example')
self.cmd.run('--add', 'Bitergia')
self.cmd.run('-d', 'Bitergia')
self.cmd.run('--add', '*****@*****.**')
self.cmd.run('-a', '*****@*****.**')
self.cmd.run('--delete', 'John Doe')
self.cmd.run('--delete', '*****@*****.**')
self.cmd.run('--add', '*****@*****.**')
self.cmd.run('-d', '*****@*****.**')
self.cmd.run('--delete', 'Example')
self.cmd.run()
output = sys.stdout.getvalue().strip()
self.assertEqual(output, BLACKLIST_OUTPUT_ALT)
def __load_test_dataset(self):
self.cmd.add('*****@*****.**')
self.cmd.add('John Smith')
self.cmd.add('Bitergia')
self.cmd.add('John Doe')
class TestDelete(unittest.TestCase):
"""Blacklist delete sub-command unit tests"""
def setUp(self):
if not hasattr(sys.stdout, 'getvalue'):
self.fail('This test needs to be run in buffered mode')
# Create a connection to check the contents of the registry
self.db = Database(DB_USER, DB_PASSWORD, DB_NAME, DB_HOST, DB_PORT)
# Create command
self.kwargs = {'user' : DB_USER,
'password' : DB_PASSWORD,
'database' :DB_NAME,
'host' : DB_HOST,
'port' : DB_PORT}
self.cmd = Blacklist(**self.kwargs)
def tearDown(self):
self.db.clear()
def test_delete(self):
"""Check whether everything works ok when deleting entries"""
# First, add a set of entries
self.cmd.add('*****@*****.**')
self.cmd.add('John Smith')
self.cmd.add('Bitergia')
self.cmd.add('John Doe')
# Delete an entry
bl = api.blacklist(self.db, 'Bitergia')
self.assertEqual(len(bl), 1)
code = self.cmd.delete('Bitergia')
self.assertEqual(code, CMD_SUCCESS)
self.assertRaises(NotFoundError, api.blacklist,
self.db, 'Bitergia')
code = self.cmd.delete('*****@*****.**')
self.assertEqual(code, CMD_SUCCESS)
# The final content of the registry should have
# two entries
bl = api.blacklist(self.db)
self.assertEqual(len(bl), 2)
e = bl[0]
self.assertEqual(e.excluded, 'John Doe')
e = bl[1]
self.assertEqual(e.excluded, 'John Smith')
def test_not_found_entry(self):
"""Check if it fails removing an entry that does not exists"""
# It should print an error when the blacklist is empty
code = self.cmd.delete('*****@*****.**')
self.assertEqual(code, CODE_NOT_FOUND_ERROR)
output = sys.stderr.getvalue().strip().split('\n')[0]
self.assertEqual(output, BLACKLIST_NOT_FOUND_ERROR)
# Add a pair of entries to check delete with a blacklist
# with contents
self.cmd.add('John Smith')
self.cmd.add('*****@*****.**')
# The error should be the same
code = self.cmd.delete('*****@*****.**')
self.assertEqual(code, CODE_NOT_FOUND_ERROR)
output = sys.stderr.getvalue().strip().split('\n')[1]
self.assertEqual(output, BLACKLIST_NOT_FOUND_ERROR)
# Nothing has been deleted from the registry
bl = api.blacklist(self.db)
self.assertEqual(len(bl), 2)
class TestBaseCase(unittest.TestCase):
"""Defines common setup and teardown methods on show unit tests"""
def setUp(self):
if not hasattr(sys.stdout, 'getvalue'):
self.fail('This test needs to be run in buffered mode')
# Create a connection to check the contents of the registry
self.db = Database(DB_USER, DB_PASSWORD, DB_NAME, DB_HOST, DB_PORT)
# Import predefined dataset for testing
self._load_test_dataset()
# Create command
self.kwargs = {'user' : DB_USER,
'password' : DB_PASSWORD,
'database' :DB_NAME,
'host' : DB_HOST,
'port' : DB_PORT}
self.cmd = Show(**self.kwargs)
def tearDown(self):
self.db.clear()
def _load_test_dataset(self):
# Add country
with self.db.connect() as session:
# Add a country
us = Country(code='US', name='United States of America', alpha3='USA')
session.add(us)
# Add organizations
api.add_organization(self.db, 'Example')
api.add_organization(self.db, 'Bitergia')
# Add John Smith identity
jsmith_uuid = api.add_identity(self.db, 'scm', '*****@*****.**',
'John Smith', 'jsmith')
api.add_identity(self.db, 'scm', '*****@*****.**', 'John Smith',
uuid=jsmith_uuid)
api.edit_profile(self.db, jsmith_uuid, email='*****@*****.**', is_bot=True)
# Add Joe Roe identity
jroe_uuid = api.add_identity(self.db, 'scm', '*****@*****.**',
'Jane Roe', 'jroe')
api.add_identity(self.db, 'scm', '*****@*****.**',
uuid=jroe_uuid)
api.add_identity(self.db, 'unknown', '*****@*****.**',
uuid=jroe_uuid)
api.edit_profile(self.db, jroe_uuid, name='Jane Roe', email='*****@*****.**',
is_bot=False, country_code='US')
# Add unique identity, this one won't have neither identities
# nor enrollments
api.add_unique_identity(self.db,
'0000000000000000000000000000000000000000')
# Add enrollments
api.add_enrollment(self.db, jsmith_uuid, 'Example')
api.add_enrollment(self.db, jroe_uuid, 'Example')
api.add_enrollment(self.db, jroe_uuid, 'Bitergia',
datetime.datetime(1999, 1, 1),
datetime.datetime(2000, 1, 1))
api.add_enrollment(self.db, jroe_uuid, 'Bitergia',
datetime.datetime(2006, 1, 1),
datetime.datetime(2008, 1, 1))
def tearDownClass(cls):
if cls.create:
Database.drop(**cls.db_kwargs)
class TestBaseCase(unittest.TestCase):
"""Defines common setup and teardown methods on show unit tests"""
def setUp(self):
import tempfile
if not hasattr(sys.stdout, 'getvalue'):
self.fail('This test needs to be run in buffered mode')
# Create a connection to check the contents of the registry
self.db = Database(DB_USER, DB_PASSWORD, DB_NAME, DB_HOST, DB_PORT)
# Import predefined dataset for testing
self._load_test_dataset()
# Temporary file for outputs
self.tmpfile = tempfile.mkstemp()[1]
# Create command
self.kwargs = {'user' : DB_USER,
'password' : DB_PASSWORD,
'database' : DB_NAME,
'host' : DB_HOST,
'port' : DB_PORT}
self.cmd = Export(**self.kwargs)
def tearDown(self):
import os
self.db.clear()
os.remove(self.tmpfile)
def read_json(self, filename):
if sys.version_info[0] >= 3: # Python 3
with open(filename, 'r', encoding='UTF-8') as f:
content = f.read()
else: # Python 2
with open(filename, 'r') as f:
content = f.read().decode('UTF-8')
obj = json.loads(content)
return obj
def _load_test_dataset(self):
import datetime
self.db.clear()
# Add country
with self.db.connect() as session:
# Add a country
us = Country(code='US', name='United States of America', alpha3='USA')
session.add(us)
# Add organizations
api.add_organization(self.db, 'Example')
api.add_domain(self.db, 'Example', 'example.com', is_top_domain=True)
api.add_domain(self.db, 'Example', 'example.net', is_top_domain=True)
api.add_organization(self.db, 'Bitergia')
api.add_domain(self.db, 'Bitergia', 'bitergia.net', is_top_domain=True)
api.add_domain(self.db, 'Bitergia', 'bitergia.com', is_top_domain=True)
api.add_domain(self.db, 'Bitergia', 'api.bitergia.com', is_top_domain=False)
api.add_domain(self.db, 'Bitergia', 'test.bitergia.com', is_top_domain=False)
api.add_organization(self.db, 'Unknown')
# Add John Smith identity
jsmith_uuid = api.add_identity(self.db, 'scm', '*****@*****.**',
'John Smith', 'jsmith')
api.add_identity(self.db, 'scm', '*****@*****.**', 'John Smith',
uuid=jsmith_uuid)
api.edit_profile(self.db, jsmith_uuid, email='*****@*****.**', is_bot=True)
# Add Joe Roe identity
jroe_uuid = api.add_identity(self.db, 'scm', '*****@*****.**',
'Jane Roe', 'jroe')
api.add_identity(self.db, 'scm', '*****@*****.**',
uuid=jroe_uuid)
api.add_identity(self.db, 'unknown', '*****@*****.**',
uuid=jroe_uuid)
api.edit_profile(self.db, jroe_uuid, name='Jane Roe', email='*****@*****.**',
is_bot=False, country_code='US')
# Add unique identity, this one won't have neither identities
# nor enrollments
api.add_unique_identity(self.db,
'0000000000000000000000000000000000000000')
# Add enrollments
api.add_enrollment(self.db, jsmith_uuid, 'Example')
api.add_enrollment(self.db, jroe_uuid, 'Example')
api.add_enrollment(self.db, jroe_uuid, 'Bitergia',
datetime.datetime(1999, 1, 1),
datetime.datetime(2000, 1, 1))
api.add_enrollment(self.db, jroe_uuid, 'Bitergia',
datetime.datetime(2006, 1, 1),
datetime.datetime(2008, 1, 1))
# Add blacklist
api.add_to_matching_blacklist(self.db, '*****@*****.**')
api.add_to_matching_blacklist(self.db, 'John Smith')
def tearDown(self):
Database.drop(database=self.name, **self.kwargs)
Database.drop(database=self.name_reuse, **self.kwargs)
class TestOrgsCommand(unittest.TestCase):
"""Organization command unit tests"""
def setUp(self):
if not hasattr(sys.stdout, 'getvalue'):
self.fail('This test needs to be run in buffered mode')
# Create a connection to check the contents of the registry
self.db = Database(DB_USER, DB_PASSWORD, DB_NAME, DB_HOST, DB_PORT)
# Create command
self.kwargs = {'user' : DB_USER,
'password' : DB_PASSWORD,
'database' :DB_NAME,
'host' : DB_HOST,
'port' : DB_PORT}
self.cmd = Organizations(**self.kwargs)
def tearDown(self):
self.db.clear()
def test_default_action(self):
"""Check whether when no action is given it runs --list"""
# Add some contents first
self.cmd.add('Example')
self.cmd.add('Example', 'example.com')
self.cmd.add('Example', 'example.org')
self.cmd.add('Example', 'example.net')
self.cmd.add('MyExample')
self.cmd.add('MyExample', 'myexample.com')
code = self.cmd.run()
self.assertEqual(code, CMD_SUCCESS)
output = sys.stdout.getvalue().strip()
self.assertEqual(output, REGISTRY_OUTPUT_EXAMPLE)
def test_list_without_args(self):
"""Test list action with and without arguments"""
self.__load_test_dataset()
code = self.cmd.run('-l')
self.assertEqual(code, CMD_SUCCESS)
output = sys.stdout.getvalue().strip()
self.assertEqual(output, REGISTRY_OUTPUT)
def test_list_with_args(self):
"""Test list action with arguments"""
self.__load_test_dataset()
# Add an extra organization
self.cmd.add('MyExample')
self.cmd.add('MyExample', 'myexample.com')
code = self.cmd.run('--list', 'Example')
self.assertEqual(code, CMD_SUCCESS)
output = sys.stdout.getvalue().strip()
self.assertEqual(output, REGISTRY_OUTPUT_EXAMPLE)
def test_add_with_args(self):
"""Test add action"""
code = self.cmd.run('--add', 'LibreSoft')
self.assertEqual(code, CMD_SUCCESS)
code = self.cmd.run('-a', 'Example')
self.assertEqual(code, CMD_SUCCESS)
code = self.cmd.run('--add', 'Example', 'example.com')
self.assertEqual(code, CMD_SUCCESS)
code = self.cmd.run('--add', 'Bitergia')
self.assertEqual(code, CMD_SUCCESS)
code = self.cmd.run('-a', 'Bitergia', 'bitergia.net')
self.assertEqual(code, CMD_SUCCESS)
code = self.cmd.run('--add', 'Example', 'example.org')
self.assertEqual(code, CMD_SUCCESS)
code = self.cmd.run('--add', 'Bitergia', 'bitergia.com', '--top-domain')
self.assertEqual(code, CMD_SUCCESS)
code = self.cmd.run('-a', 'Example', 'example.net')
self.assertEqual(code, CMD_SUCCESS)
self.cmd.run('--list')
output = sys.stdout.getvalue().strip()
self.assertEqual(output, REGISTRY_OUTPUT)
def test_add_without_args(self):
"""Check when calling --add without args, it does not do anything"""
code = self.cmd.run('--add')
self.assertEqual(code, CMD_SUCCESS)
self.cmd.run('-l')
output = sys.stdout.getvalue().strip()
self.assertEqual(output, REGISTRY_EMPTY_OUTPUT)
def test_add_with_overwrite_option(self):
"""Check whether it not fails running add with overwrite option"""
self.__load_test_dataset()
code = self.cmd.run('--add', 'Example', 'bitergia.com')
self.assertEqual(code, CODE_ALREADY_EXISTS_ERROR)
output = sys.stderr.getvalue().strip()
self.assertEqual(output, REGISTRY_DOM_ALREADY_EXISTS_ERROR)
code = self.cmd.run('--add', '--overwrite', 'Example', 'bitergia.com')
self.assertEqual(code, CMD_SUCCESS)
self.cmd.run('-l')
output = sys.stdout.getvalue().strip()
self.assertEqual(output, REGISTRY_OUTPUT_ALT)
def test_delete_with_args(self):
"""Test delete action"""
self.__load_test_dataset()
# Delete contents
code = self.cmd.run('--delete', 'Bitergia', 'bitergia.com')
self.assertEqual(code, CMD_SUCCESS)
code = self.cmd.run('-d', 'LibreSoft')
self.assertEqual(code, CMD_SUCCESS)
code = self.cmd.run('--delete', 'Bitergia')
self.assertEqual(code, CMD_SUCCESS)
code = self.cmd.run('-d', 'Example', 'example.org')
self.assertEqual(code, CMD_SUCCESS)
self.cmd.run('--list')
output = sys.stdout.getvalue().strip()
self.assertEqual(output, REGISTRY_OUTPUT_EXAMPLE_ALT)
def test_delete_without_args(self):
"""Check when calling --delete without args, it does not do anything"""
self.__load_test_dataset()
code = self.cmd.run('--delete')
self.assertEqual(code, CMD_SUCCESS)
self.cmd.run('-l')
output = sys.stdout.getvalue().strip()
self.assertEqual(output, REGISTRY_OUTPUT)
def test_run_mixing_actions(self):
"""Check how it works when mixing actions"""
self.cmd.run('--add', 'LibreSoft')
self.cmd.run('-a', 'LibreSoft', 'libresoft.es')
self.cmd.run('-a', 'Example')
self.cmd.run('--add', 'Example', 'example.org')
self.cmd.run('-d', 'Example', 'example.org')
self.cmd.run('--add', 'Bitergia')
self.cmd.run('-a', 'Bitergia', 'bitergia.net')
self.cmd.run('--delete', 'LibreSoft')
self.cmd.run('--add', 'Example', 'example.com')
self.cmd.run('--add', 'Bitergia', 'bitergia.com')
self.cmd.run('-a', 'Example', 'example.net')
self.cmd.run('--delete', 'Bitergia', 'bitergia.com')
self.cmd.run('-d', 'Bitergia')
self.cmd.run()
output = sys.stdout.getvalue().strip()
self.assertEqual(output, REGISTRY_OUTPUT_EXAMPLE_ALT)
def __load_test_dataset(self):
self.cmd.add('Example')
self.cmd.add('Example', 'example.com')
self.cmd.add('Bitergia')
self.cmd.add('Bitergia', 'bitergia.net')
self.cmd.add('Bitergia', 'bitergia.com', is_top_domain=True)
self.cmd.add('LibreSoft')
self.cmd.add('Example', 'example.org')
self.cmd.add('Example', 'example.net')
class TestAdd(unittest.TestCase):
"""Blacklist add sub-command unit tests"""
def setUp(self):
if not hasattr(sys.stdout, 'getvalue'):
self.fail('This test needs to be run in buffered mode')
# Create a connection to check the contents of the registry
self.db = Database(DB_USER, DB_PASSWORD, DB_NAME, DB_HOST, DB_PORT)
# Create command
self.kwargs = {'user' : DB_USER,
'password' : DB_PASSWORD,
'database' :DB_NAME,
'host' : DB_HOST,
'port' : DB_PORT}
self.cmd = Blacklist(**self.kwargs)
def tearDown(self):
self.db.clear()
def test_add(self):
"""Check whether everything works ok when adding entries"""
self.cmd.add('*****@*****.**')
self.cmd.add('John Smith')
self.cmd.add('Bitergia')
self.cmd.add('John Doe')
# List the registry and check the output
code = self.cmd.blacklist()
self.assertEqual(code, CMD_SUCCESS)
output = sys.stdout.getvalue().strip()
self.assertEqual(output, BLACKLIST_OUTPUT)
def test_existing_entry(self):
"""Check if it fails adding an entry that already exists"""
code = self.cmd.add('*****@*****.**')
self.assertEqual(code, CMD_SUCCESS)
code = self.cmd.add('*****@*****.**')
self.assertEqual(code, CODE_ALREADY_EXISTS_ERROR)
output = sys.stderr.getvalue().strip()
self.assertEqual(output, BLACKLIST_ALREADY_EXISTS_ERROR)
def test_none_entry(self):
"""Check behavior adding None entries"""
code = self.cmd.add(None)
self.assertEqual(code, CMD_SUCCESS)
output = sys.stdout.getvalue().strip()
self.assertEqual(output, BLACKLIST_EMPTY_OUTPUT)
# The blacklist should be empty
bl = api.blacklist(self.db)
self.assertEqual(len(bl), 0)
def test_empty_entry(self):
"""Check behavior adding empty organizations"""
code = self.cmd.add('')
self.assertEqual(code, CMD_SUCCESS)
output = sys.stdout.getvalue().strip()
self.assertEqual(output, BLACKLIST_EMPTY_OUTPUT)
# The blacklist should be empty
bl = api.blacklist(self.db)
self.assertEqual(len(bl), 0)
def tearDown(self):
Database.drop(self.kwargs['user'], self.kwargs['password'],
self.name, self.kwargs['host'],
self.kwargs['port'])
class TaskIdentitiesMerge(Task):
""" Task for processing identities in SortingHat """
def __init__(self, conf):
super().__init__(conf)
self.sh_kwargs = {'user': self.db_user, 'password': self.db_password,
'database': self.db_sh, 'host': self.db_host,
'port': None}
self.db = Database(**self.sh_kwargs)
self.last_autorefresh = datetime.utcnow() # Last autorefresh date
def is_backend_task(self):
return False
def __get_uuids_from_profile_name(self, profile_name):
""" Get the uuid for a profile name """
uuids = []
with self.db.connect() as session:
query = session.query(Profile).\
filter(Profile.name == profile_name)
profiles = query.all()
if profiles:
for p in profiles:
uuids.append(p.uuid)
return uuids
def __build_sh_command(self):
cfg = self.config.get_conf()
db_user = cfg['sortinghat']['user']
db_password = cfg['sortinghat']['password']
db_host = cfg['sortinghat']['host']
db_name = cfg['sortinghat']['database']
cmd = ['sortinghat', '-u', db_user, '-p', db_password, '--host', db_host,
'-d', db_name]
return cmd
def __execute_sh_command(self, cmd):
logger.debug("Executing %s", cmd)
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
outs, errs = proc.communicate()
return_code = proc.returncode
if return_code != 0:
logger.error("[sortinghat] Error in command %s", cmd)
return return_code
def do_affiliate(self):
cmd = self.__build_sh_command()
cmd += ['affiliate']
self.__execute_sh_command(cmd)
return
def do_autogender(self):
cmd = self.__build_sh_command()
cmd += ['autogender']
self.__execute_sh_command(cmd)
return None
def do_autoprofile(self, sources):
cmd = self.__build_sh_command()
cmd += ['autoprofile'] + sources
self.__execute_sh_command(cmd)
return None
def do_unify(self, kwargs):
cmd = self.__build_sh_command()
cmd += ['unify', '--fast-matching', '-m', kwargs['matching']]
if not kwargs['strict_mapping']:
cmd += ['--no-strict-matching']
self.__execute_sh_command(cmd)
return
def execute(self):
# ** START SYNC LOGIC **
# Check that enrichment tasks are not active before loading identities
while True:
time.sleep(1) # check each second if the task could start
with TasksManager.IDENTITIES_TASKS_ON_LOCK:
with TasksManager.NUMBER_ENRICH_TASKS_ON_LOCK:
enrich_tasks = TasksManager.NUMBER_ENRICH_TASKS_ON
logger.debug("[unify] Enrich tasks active: %i", enrich_tasks)
if enrich_tasks == 0:
# The load of identities can be started
TasksManager.IDENTITIES_TASKS_ON = True
break
# ** END SYNC LOGIC **
cfg = self.config.get_conf()
uuids_refresh = []
for algo in cfg['sortinghat']['matching']:
if not algo:
# cfg['sortinghat']['matching'] is an empty list
logger.debug('Unify not executed because empty algorithm')
continue
kwargs = {'matching': algo, 'fast_matching': True,
'strict_mapping': cfg['sortinghat']['strict_mapping']}
logger.info("[sortinghat] Unifying identities using algorithm %s",
kwargs['matching'])
self.do_unify(kwargs)
if not cfg['sortinghat']['affiliate']:
logger.debug("Not doing affiliation")
else:
# Global enrollments using domains
logger.info("[sortinghat] Executing affiliate")
self.do_affiliate()
if 'autoprofile' not in cfg['sortinghat'] or \
not cfg['sortinghat']['autoprofile'][0]:
logger.info("[sortinghat] Autoprofile not configured. Skipping.")
else:
logger.info("[sortinghat] Executing autoprofile for sources: %s",
cfg['sortinghat']['autoprofile'])
sources = cfg['sortinghat']['autoprofile']
self.do_autoprofile(sources)
if 'autogender' not in cfg['sortinghat'] or \
not cfg['sortinghat']['autogender']:
logger.info("[sortinghat] Autogender not configured. Skipping.")
else:
logger.info("[sortinghat] Executing autogender")
self.do_autogender()
if 'bots_names' not in cfg['sortinghat']:
logger.info("[sortinghat] Bots name list not configured. Skipping.")
else:
logger.info("[sortinghat] Marking bots: %s",
cfg['sortinghat']['bots_names'])
for name in cfg['sortinghat']['bots_names']:
# First we need the uuids for the profile name
uuids = self.__get_uuids_from_profile_name(name)
# Then we can modify the profile setting bot flag
profile = {"is_bot": True}
for uuid in uuids:
api.edit_profile(self.db, uuid, **profile)
# For quitting the bot flag - debug feature
if 'no_bots_names' in cfg['sortinghat']:
logger.info("[sortinghat] Removing Marking bots: %s",
cfg['sortinghat']['no_bots_names'])
for name in cfg['sortinghat']['no_bots_names']:
uuids = self.__get_uuids_from_profile_name(name)
profile = {"is_bot": False}
for uuid in uuids:
api.edit_profile(self.db, uuid, **profile)
with TasksManager.IDENTITIES_TASKS_ON_LOCK:
TasksManager.IDENTITIES_TASKS_ON = False
