def check_config(cls, config):
# First let's check all common sections entries
check_params = cls.general_params()
backend_sections = cls.get_backend_sections()
study_sections = cls.get_study_sections()
# filter out commented sections (e.g., [*backend_section:tag])
config_sections = [
section for section in config.keys()
if section.split(":")[0][0] != "*"
]
for section in config_sections:
if Task.get_backend(section) in backend_sections:
# backend_section:tag
continue
if section.startswith((study_sections)):
continue
if section not in check_params.keys():
raise RuntimeError("Wrong section:", section)
# Check the params for the section
for param in config[section].keys():
if param not in check_params[section]:
raise RuntimeError("Wrong section param:", section, param)
for param in check_params[section]:
if param not in config[section].keys():
if not check_params[section][param]['optional']:
raise RuntimeError("Missing section param:", section,
param)
else:
# Add the default value for this param
config[section][param] = check_params[section][param][
'default']
else:
ptype = type(config[section][param])
ptype_ok = check_params[section][param]["type"]
ptype_default = check_params[section][param]["default"]
if ptype != ptype_ok and ptype_default is not None:
msg = "Wrong type for section param: %s %s %s should be %s" % \
(section, param, ptype, ptype_ok)
raise RuntimeError(msg)
# And now the backend_section entries
# A backend section entry could have specific perceval params which are
# not checked
check_params = cls.backend_section_params()
for section in config_sections:
if Task.get_backend(section) in backend_sections:
for param in check_params:
if param not in config[section].keys():
if not check_params[param]['optional']:
raise RuntimeError("Missing section param:",
section, param)
else:
ptype = type(config[section][param])
ptype_ok = check_params[param]["type"]
if ptype != ptype_ok:
msg = "Wrong type for section param: %s %s %s should be %s" % \
(section, param, ptype, ptype_ok)
raise RuntimeError(msg)
def test_compose_p2o_params(self):
"""Test whether p2o params are built correctly for a backend and a repository"""
config = Config(CONF_FILE)
task = Task(config)
params = task._compose_p2o_params(
"stackexchange",
"https://stackoverflow.com/questions/tagged/example")
self.assertDictEqual(
params,
{'url': "https://stackoverflow.com/questions/tagged/example"})
params = task._compose_p2o_params(
"mediawiki", "https://wiki-archive.opendaylight.org "
"https://wiki-archive.opendaylight.org/view")
self.assertDictEqual(
params, {
'url':
"https://wiki-archive.opendaylight.org "
"https://wiki-archive.opendaylight.org/view"
})
params = task._compose_p2o_params(
"mediawiki", "https://wiki-archive.opendaylight.org "
"https://wiki-archive.opendaylight.org/view "
"--filter-no-collection=true")
self.assertDictEqual(
params, {
'url': "https://wiki-archive.opendaylight.org "
"https://wiki-archive.opendaylight.org/view",
"filter-no-collection": "true"
})
def test_compose_p2o_params(self):
"""Test whether p2o params are built correctly for a backend and a repository"""
config = Config(CONF_FILE)
task = Task(config)
params = task._compose_p2o_params("stackexchange", "https://stackoverflow.com/questions/tagged/example")
self.assertEqual(params, {'url': "https://stackoverflow.com/questions/tagged/example"})
def test_get_collection_url(self):
"""Test whether the collection url could be overwritten in a backend"""
config = Config(CONF_FILE)
task = Task(config)
task.backend_section = "stackexchange"
self.assertEqual(task._get_collection_url(), COLLECTION_URL_STACKEXCHANGE)
def check_config(cls, config):
# First let's check all common sections entries
check_params = cls.general_params()
backend_sections = cls.get_backend_sections()
study_sections = cls.get_study_sections()
# filter out commented sections (e.g., [*backend_section:tag])
config_sections = [section for section in config.keys() if section.split(":")[0][0] != "*"]
for section in config_sections:
if Task.get_backend(section) in backend_sections:
# backend_section:tag
continue
if section.startswith((study_sections)):
continue
if section not in check_params.keys():
raise RuntimeError("Wrong section:", section)
# Check the params for the section
for param in config[section].keys():
if param not in check_params[section]:
raise RuntimeError("Wrong section param:", section, param)
for param in check_params[section]:
if param not in config[section].keys():
if not check_params[section][param]['optional']:
raise RuntimeError("Missing section param:", section, param)
else:
# Add the default value for this param
config[section][param] = check_params[section][param]['default']
else:
ptype = type(config[section][param])
ptype_ok = check_params[section][param]["type"]
ptype_default = check_params[section][param]["default"]
if ptype != ptype_ok and ptype_default is not None:
msg = "Wrong type for section param: %s %s %s should be %s" % \
(section, param, ptype, ptype_ok)
raise RuntimeError(msg)
# And now the backend_section entries
# This only validates the types of each param if present, and doesn't check that
# all required parameters are set. This functionality has been moved to the
# get_backend_section method
check_params = cls.backend_section_params()
for section in config_sections:
if Task.get_backend(section) in backend_sections:
for param in check_params:
if param in config[section].keys():
ptype = type(config[section][param])
ptype_ok = check_params[param]["type"]
if ptype != ptype_ok:
msg = "Wrong type for section param: %s %s %s should be %s" % \
(section, param, ptype, ptype_ok)
raise RuntimeError(msg)
def test_compose_perceval_params(self):
"""Test whether perceval params are built correctly for a backend and a repository"""
expected_repo_params = json.loads(read_file('data/task-params-expected'))
config = Config(CONF_FILE)
task = Task(config)
for backend in expected_repo_params.keys():
repo = expected_repo_params.get(backend)['repo']
perceval_params = task._compose_perceval_params(backend, repo)
expected_params = expected_repo_params.get(backend)['params']
self.assertEqual(expected_params.sort(), perceval_params.sort())
def _get_repos_by_backend(self):
#
# return dict with backend and list of repositories
#
output = {}
projects = TaskProjects.get_projects()
for backend_section in Config.get_backend_sections():
for pro in projects:
backend = Task.get_backend(backend_section)
if backend in projects[pro]:
if backend_section not in output:
output[backend_section] = projects[pro][backend]
else:
output[backend_section] += projects[pro][backend]
# backend could be in project/repo file but not enabled in
# sirmordred conf file
enabled = {}
for k in output:
if k in self.conf:
enabled[k] = output[k]
# logger.debug('repos to be retrieved: %s ', enabled)
return enabled
def test_extract_repo_tags(self):
"""Test the extraction of tags in repositories"""
config = Config(CONF_FILE)
task = Task(config)
url, tags = task._extract_repo_tags("git", "https://github.com/zhquan_example/repo --labels=[ENG, SUPP]")
self.assertEqual(url, "https://github.com/zhquan_example/repo")
self.assertListEqual(tags, ["ENG", "SUPP"])
# By default it extracts '--labels'
url, tags = task._extract_repo_tags("confluence", "https://example.com --spaces=[HOME]")
self.assertEqual(url, "https://example.com --spaces=[HOME]")
self.assertListEqual(tags, [])
# To extracts '--spaces' add tag_type='spaces'
url, tags = task._extract_repo_tags("confluence", "https://example.com --spaces=[HOME]", tag_type="spaces")
self.assertEqual(url, "https://example.com")
self.assertListEqual(tags, ["HOME"])
def test_initialization(self):
"""Test whether attributes are initializated"""
config = Config(CONF_FILE)
task = Task(config)
self.assertEqual(task.config, config)
self.assertEqual(task.db_sh, task.conf['sortinghat']['database'])
self.assertEqual(task.db_user, task.conf['sortinghat']['user'])
self.assertEqual(task.db_password, task.conf['sortinghat']['password'])
self.assertEqual(task.db_host, task.conf['sortinghat']['host'])
def get_repos_by_backend_section(cls, backend_section):
""" return list with the repositories for a backend_section """
repos = []
projects = TaskProjects.get_projects()
for pro in projects:
if backend_section in projects[pro]:
backend = Task.get_backend(backend_section)
if backend in Config.get_global_data_sources() and cls.GLOBAL_PROJECT in projects \
and pro != cls.GLOBAL_PROJECT:
logger.debug("Skip global data source %s for project %s",
backend, pro)
else:
repos += projects[pro][backend_section]
logger.debug("List of repos for %s: %s", backend_section, repos)
return repos
def test_run(self):
"""Test whether the Task could be run"""
config = Config(CONF_FILE)
task = Task(config)
self.assertEqual(task.execute(), None)
