def test__get_projects_from_url(self):
"""Test downloading projects from an URL """
setup_http_server()
projects_url = 'http://localhost/projects.json'
config = Config(CONF_FILE)
config.set_param('projects', 'projects_url', projects_url)
task = TaskProjects(config)
self.assertEqual(task.execute(), None)
projects = task.get_projects()
self.assertTrue(URL_PROJECTS_MAIN in projects)
def setUp(self):
config = Config(CONF_FILE)
sh = config.get_conf()['sortinghat']
self.sh_kwargs = {'user': sh['user'], 'password': sh['password'],
'database': sh['database'], 'host': sh['host'],
'port': None}
# Clean the database to start an empty state
Database.drop(**self.sh_kwargs)
# Create command
Database.create(**self.sh_kwargs)
self.sh_db = Database(**self.sh_kwargs)
def _get_repos_by_backend(self):
#
# return dict with backend and list of repositories
#
output = {}
projects = TaskProjects.get_projects()
for backend_section in Config.get_backend_sections():
for pro in projects:
backend = Task.get_backend(backend_section)
if backend in projects[pro]:
if backend_section not in output:
output[backend_section] = projects[pro][backend]
else:
output[backend_section] += projects[pro][backend]
# backend could be in project/repo file but not enabled in
# mordred conf file
enabled = {}
for k in output:
if k in self.conf:
enabled[k] = output[k]
# logger.debug('repos to be retrieved: %s ', enabled)
return enabled
def test_convert_from_eclipse(self):
"""Test the conversion from eclipse projects to grimoire projects"""
setup_http_server()
projects_file = 'test-projects-eclipse.json'
config = Config(CONF_FILE)
config.set_param('projects', 'load_eclipse', True)
config.set_param('projects', 'projects_file', projects_file)
task = TaskProjects(config)
self.assertEqual(task.execute(), None)
projects = task.get_projects()
self.assertTrue(TaskProjects.GLOBAL_PROJECT in projects)
self.assertEqual(projects['birt']['github'][0], 'https://github.com/eclipse/birt')
remove(projects_file)
def setUp(self):
config = Config(CONF_FILE)
sh = config.get_conf()['sortinghat']
self.sh_kwargs = {
'user': sh['user'],
'password': sh['password'],
'database': sh['database'],
'host': sh['host'],
'port': None
}
# Clean the database to start an empty state
Database.drop(**self.sh_kwargs)
# Create command
Database.create(**self.sh_kwargs)
self.sh_db = Database(**self.sh_kwargs)
def test_identities_load_file(self):
""" Check the local loading of identities files """
setup_http_server()
config = Config(CONF_FILE)
task = TaskIdentitiesLoad(config)
task.execute()
# Check the number of identities loaded from local and remote files
nuids = len(api.unique_identities(self.sh_db))
self.assertEqual(nuids, 4)
def off_test_load_orgs(self):
""" Test loading of orgs in SH """
setup_http_server()
config = Config(CONF_FILE)
task = TaskIdentitiesLoad(config)
task.execute()
# Check the number of orgs loaded
norgs = len(api.registry(self.sh_db))
self.assertEqual(norgs, 20)
def test_get_collection_url(self):
"""Test whether the collection url could be overwritten in a backend"""
config = Config(CONF_FILE)
task = Task(config)
task.backend_section = "stackexchange"
self.assertEqual(task.conf['es_collection']['url'], COLLECTION_URL)
self.assertEqual(task._get_collection_url(),
COLLECTION_URL_STACKEXCHANGE)
def setUp(self):
self.config = Config(CONF_FILE)
mordred = Mordred(self.config)
task = TaskProjects(self.config)
self.assertEqual(task.execute(), None)
self.backends = mordred._get_repos_by_backend()
self.backend_tasks = [TaskRawDataCollection, TaskEnrich]
self.stopper = threading.Event()
def test_initialization(self):
"""Test whether attributes are initializated"""
config = Config(CONF_FILE)
task = Task(config)
self.assertEqual(task.config, config)
self.assertEqual(task.db_sh, task.conf['sortinghat']['database'])
self.assertEqual(task.db_user, task.conf['sortinghat']['user'])
self.assertEqual(task.db_password, task.conf['sortinghat']['password'])
self.assertEqual(task.db_host, task.conf['sortinghat']['host'])
def test_compose_p2o_params(self):
"""Test whether p2o params are built correctly for a backend and a repository"""
config = Config(CONF_FILE)
task = Task(config)
params = task._compose_p2o_params(
"stackexchange",
"https://stackoverflow.com/questions/tagged/example")
self.assertEqual(
params,
{'url': "https://stackoverflow.com/questions/tagged/example"})
def test_run_eclipse(self):
"""Test whether the Task could be run getting projects from Eclipse"""
setup_http_server()
# Create a empty projects file for testing
projects_file = 'test-projects-eclipse.json'
config = Config(CONF_FILE)
config.set_param('projects', 'load_eclipse', True)
config.set_param('projects', 'projects_file', projects_file)
task = TaskProjects(config)
self.assertEqual(task.execute(), None)
self.assertEqual(len(task.get_projects().keys()), 302)
# Let's remove some projects to track changes
with open(ECLIPSE_PROJECTS_FILE) as eproj:
remove_project = 'birt'
add_project = 'new_project'
new_projects = task.convert_from_eclipse(json.load(eproj)['projects'])
new_projects.pop(remove_project)
new_projects.update({add_project: {}})
task.set_projects(new_projects)
self.assertEqual(task.get_projects_last_diff().sort(),
[add_project, remove_project].sort())
remove(projects_file)
def test_studies(self):
"""Test whether the studies configuration works """
config = Config(CONF_FILE)
cfg = config.get_conf()
# We need to load the projects
TaskProjects(config).execute()
backend_section = GIT_BACKEND_SECTION
task = TaskEnrich(config, backend_section=backend_section)
# Configure no studies
cfg['git']['studies'] = None
self.assertEqual(task.execute(), None)
# Configure no studies
cfg['git']['studies'] = []
self.assertEqual(task.execute(), None)
# Configure a wrong study
cfg['git']['studies'] = ['bad_study']
with self.assertRaises(DataEnrichmentError):
self.assertEqual(task.execute(), None)
# Configure several studies
cfg['git']['studies'] = ['enrich_onion']
self.assertEqual(task.execute(), None)
# Configure several studies
cfg['git']['studies'] = ['enrich_demography:1', 'enrich_areas_of_code']
self.assertEqual(task.execute(), None)
# Configure kafka kip study
cfg['mbox']['studies'] = ['kafka_kip']
self.assertEqual(task.execute(), None)
# Configure several studies, one wrong
cfg['git']['studies'] = [
'enrich_demography:1', "enrich_areas_of_code1"
]
with self.assertRaises(DataEnrichmentError):
self.assertEqual(task.execute(), None)
def test_compose_perceval_params(self):
"""Test whether perceval params are built correctly for a backend and a repository"""
config = Config(CONF_FILE)
task = Task(config)
params = [
'--site', 'stackoverflow.com', '--tagged', 'ovirt', '--tag',
'https://stackoverflow.com/questions/tagged/ovirt', '--api-token',
'token', '--fetch-cache'
]
perceval_params = task._compose_perceval_params(
BACKEND_NAME, REPO_NAME)
self.assertEqual(params.sort(), perceval_params.sort())
def test_autogender(self):
"""Test whether autogender SH command is executed"""
config = Config(CONF_FILE)
# Test default value
self.assertEqual(config.get_conf()['sortinghat']['gender'], False)
config.get_conf()['sortinghat']['gender'] = True
# Load some identities
task = TaskIdentitiesLoad(config)
task.execute()
# Check the number of identities loaded from local and remote files
uids = api.unique_identities(self.sh_db)
task = TaskIdentitiesMerge(config)
self.assertEqual(task.do_autogender(), None)
uids = api.unique_identities(self.sh_db)
found_genders = [uid.profile.gender for uid in uids]
expected_genders = ['male', 'female', 'male', 'male']
self.assertEqual(found_genders, expected_genders)
def test_compose_perceval_params(self):
"""Test whether perceval params are built correctly for a backend and a repository"""
expected_repo_params = json.loads(
read_file('data/task-params-expected'))
config = Config(CONF_FILE)
task = Task(config)
for backend in expected_repo_params.keys():
repo = expected_repo_params.get(backend)['repo']
perceval_params = task._compose_perceval_params(backend, repo)
expected_params = expected_repo_params.get(backend)['params']
self.assertEqual(expected_params.sort(), perceval_params.sort())
def get_repos_by_backend_section(cls, backend_section):
""" return list with the repositories for a backend_section """
repos = []
backend = Task.get_backend(backend_section)
projects = TaskProjects.get_projects()
for pro in projects:
if backend in projects[pro]:
if (backend in Config.get_global_data_sources() and
cls.GLOBAL_PROJECT in projects and pro != cls.GLOBAL_PROJECT):
logger.debug("Skip global data source %s for project %s",
backend, pro)
else:
repos += projects[pro][backend]
logger.debug("List of repos for %s: %s", backend_section, repos)
return repos
def get_repos_by_backend_section(cls, backend_section):
""" return list with the repositories for a backend_section """
repos = []
backend = Task.get_backend(backend_section)
projects = TaskProjects.get_projects()
for pro in projects:
if backend in projects[pro]:
if (backend in Config.get_global_data_sources() and
cls.GLOBAL_PROJECT in projects and pro != cls.GLOBAL_PROJECT):
logger.debug("Skip global data source %s for project %s",
backend, pro)
else:
repos += projects[pro][backend]
logger.debug("List of repos for %s: %s", backend_section, repos)
return repos
def test_backend_params(self):
"""Test whether the backend parameters are initializated"""
config = Config(CONF_FILE)
backend_section = GITHUB_BACKEND_SECTION
task = TaskRawDataCollection(config, backend_section=backend_section)
params = task._compose_perceval_params(GITHUB_BACKEND_SECTION,
GITHUB_REPO)
expected_params = [
'grimoirelab', 'perceval', '--api-token', 'XXXXX', '--sleep-time',
'300', '--sleep-for-rate', '--category', 'issue', '--archive-path',
'/tmp/test_github_archive'
]
self.assertEqual(len(params), len(expected_params))
for p in params:
self.assertTrue(p in expected_params)
def test_execute_from_archive(self):
"""Test fetching data from archives"""
# proj_file -> 'test-projects-archive.json' stored within the conf file
conf_file = 'archive-test.cfg'
config = Config(conf_file)
backend_sections = [
'askbot', 'bugzilla', 'bugzillarest', 'confluence', 'discourse',
'dockerhub', 'gerrit', 'github', 'jenkins', 'jira', 'mediawiki',
'meetup', 'nntp', 'phabricator', 'redmine', 'rss', 'stackexchange',
'slack', 'telegram'
]
for backend_section in backend_sections:
task = TaskRawDataCollection(config,
backend_section=backend_section)
# We need to load the projects
TaskProjects(config).execute()
self.assertEqual(task.execute(), None)
def test_convert_from_eclipse(self):
"""Test the conversion from eclipse projects to grimoire projects"""
setup_http_server()
projects_file = 'test-projects-eclipse.json'
config = Config(CONF_FILE)
config.set_param('projects', 'load_eclipse', True)
config.set_param('projects', 'projects_file', projects_file)
task = TaskProjects(config)
self.assertEqual(task.execute(), None)
projects = task.get_projects()
self.assertTrue(TaskProjects.GLOBAL_PROJECT in projects)
remove(projects_file)
def test_run(self):
"""Test whether the Task could be run"""
config = Config(CONF_FILE)
task = Task(config)
self.assertEqual(task.execute(), None)
def test_run(self):
"""Test whether the Task could be run"""
config = Config(CONF_FILE)
task = TaskProjects(config)
self.assertEqual(task.execute(), None)
self.assertEqual(len(task.get_projects().keys()), 1)
return result
def check_panel(panel, config):
# Time to check the panel
if not os.path.isfile(panel):
logging.error("%s does not exists", panel)
return
print("Checking %s" % panel)
try:
results = cmp_panel_mapping(
es_host=config.get_conf()['es_enrichment']['url'],
panel_path=panel)
print("RESULT: ", results[list(results.keys())[0]]['status'])
except Exception as ex:
logging.error("Problem with checking: %s", ex)
if __name__ == '__main__':
config = Config(MORDRED_CFG)
panels = TaskPanels(config)
for ds in panels.panels:
for panel in panels.panels[ds]:
check_panel(panel, config)
# Check the common panels also
for panel in TaskPanels.panels_common:
check_panel(panel, config)