def test_handle_overrides_no_overrides(self):
overrides = type("Overrides", (), {})
in_config = {
'log_file': "/another/path/to/logs",
'username': "******",
'log_level': "INFO"
}
config = PromoterLegacyConfig(self.filepaths['correct'],
filters=[],
checks=[])
out_config = config.handle_overrides(in_config, overrides=overrides)
self.assertEqual(out_config, in_config)
def test_get_dlrn_api_url_local(self, check_port_mock, mock_log_debug,
mock_log_error):
check_port_mock.return_value = True
config = PromoterLegacyConfig(self.filepaths['correct'],
filters=[],
checks=[])
in_config = {}
expected_url = "http://localhost:58080"
api_url = config.get_dlrn_api_url(in_config)
self.assertEqual(api_url, expected_url)
check_port_mock.assert_has_calls([mock.call("localhost", "58080")])
mock_log_debug.assert_has_calls(
[mock.call("Assigning api_url %s", expected_url)])
self.assertFalse(mock_log_error.called)
def main(cmd_line=None):
"""
This main will gather the cli arguments and start the promoter
:param cmd_line: (optional) we can pass a string simulating a command
line string with arguments. Useful for testing the main function
:return: None
"""
args = arg_parser(cmd_line=cmd_line)
try:
common.get_lock("promoter")
except LockError:
print("Another promoter instance is running, wait for it to finish or "
"kill it and then retry")
raise
if args.config_file is not None:
# If a config file is specified use legacy config builder
config = PromoterLegacyConfig(args.config_file, overrides=args)
else:
# If not then use the config root and the new config builder
# Which is not implemented yet
raise Exception("New config method is not implemented yet")
promoter = Promoter(config)
args.handler(promoter, args)
def test_handle_overrides_some_overrides(self, mock_log_debug):
overrides = type("Overrides", (), {
'log_file': "/path/to/logs",
'api_url': "http://api.url:8080"
})
in_config = {
'log_file': "/another/path/to/logs",
'api_url': "https://localhost:389"
}
config = PromoterLegacyConfig(self.filepaths['correct'],
filters=[],
checks=[])
out_config = config.handle_overrides(in_config, overrides=overrides)
self.assertEqual(out_config['log_file'], "/path/to/logs")
self.assertEqual(out_config['api_url'], "http://api.url:8080")
mock_log_debug.assert_has_calls(
[mock.call("Main config key %s not overridden", mock.ANY)])
def test_expand_config_ussuri(self, get_api_url_mock):
api_url = "http://localhost:58080"
in_config = {
'promotion_criteria_map': {},
'release': "ussuri",
'overcloud_images': {
'qcow_servers': {
'default_server': {}
}
},
'default_qcow_server': 'default_server'
}
get_api_url_mock.return_value = api_url
config = PromoterLegacyConfig(self.filepaths['correct'],
filters=[],
checks=[])
out_config = config.expand_config(in_config)
self.assertEqual(out_config['source_namespace'], "tripleou")
self.assertEqual(out_config['target_namespace'], "tripleou")
def test_load_empty_criteria(self, mock_error):
os.environ["DLRNAPI_PASSWORD"] = "******"
with self.assertRaises(ConfigError):
PromoterLegacyConfig(self.filepaths['criteria_empty'])
calls = [
mock.call('No jobs in criteria for target %s', 'current-tripleo'),
mock.call('Error in configuration file {}'
''.format(self.filepaths['criteria_empty']))
]
mock_error.assert_has_calls(calls, any_order=True)
def test_load_ini_file_no_pass(self, mock_error):
try:
del (os.environ["DLRNAPI_PASSWORD"])
except KeyError:
pass
with self.assertRaises(ConfigError):
PromoterLegacyConfig(self.filepaths['correct'])
calls = [
mock.call('No dlrnapi password found in env'),
]
mock_error.assert_has_calls(calls, any_order=True)
def test_get_dlrn_api_url_remote_centos7_train(self, check_port_mock,
mock_log_debug,
mock_log_error):
check_port_mock.side_effect = [False, True]
config = PromoterLegacyConfig(self.filepaths['correct'],
filters=[],
checks=[])
in_config = {
'distro_name': "centos",
'distro_version': "7",
'release': "train"
}
expected_url = "https://trunk.rdoproject.org/api-centos-train"
api_url = config.get_dlrn_api_url(in_config)
self.assertEqual(api_url, expected_url)
check_port_mock.assert_has_calls([
mock.call("localhost", "58080"),
mock.call("trunk.rdoproject.org", 443, 5)
])
mock_log_debug.assert_has_calls(
[mock.call("Assigning api_url %s", expected_url)])
self.assertFalse(mock_log_error.called)
def test_get_dlrn_api_url_none(self, check_port_mock, mock_log_debug,
mock_log_error):
check_port_mock.side_effect = [False, False]
config = PromoterLegacyConfig(self.filepaths['correct'],
filters=[],
checks=[])
# PromoterConfig calls log.debug at this point, so we reset the mock
# to make it count from zero
mock_log_debug.reset_mock()
in_config = {
'distro_name': "centos",
'distro_version': "8",
'release': "master"
}
expected_url = None
api_url = config.get_dlrn_api_url(in_config)
self.assertEqual(api_url, expected_url)
check_port_mock.assert_has_calls([
mock.call("localhost", "58080"),
mock.call("trunk.rdoproject.org", 443, 5)
])
mock_log_error.assert_has_calls([mock.call("No valid API url found")])
self.assertFalse(mock_log_debug.called)
def test_sanity_check_all_checks_fail_all_errors(self, mock_log_warning,
mock_log_error):
pconfig = PromoterLegacyConfig(self.filepaths['correct'],
filters=[],
checks=[])
config = {
'distro_name': "default_distro",
'distro_version': "default_version",
'dlrnauth_password': None,
'dlrnauth_username': '******',
'promotion_criteria_map': {
'current-tripleo': []
},
'log_level': "INFO",
"log_file": "/tmp/not/existing/file.log"
}
file_config = {'main': {}}
conf_ok = pconfig.sanity_check(config, file_config)
self.assertFalse(conf_ok)
mock_log_error.assert_has_calls([
mock.call('Invalid Log file: %s', '/tmp/not/existing/file.log'),
mock.call('No dlrnapi password found in env'),
mock.call('No jobs in criteria for target %s', 'current-tripleo')
])
def test_sanity_check_all_checks_success_all_warnings(
self, mock_log_warning, mock_log_error):
pconfig = PromoterLegacyConfig(self.filepaths['correct'],
filters=[],
checks=[])
config = {
'distro_name': "default_distro",
'distro_version': "default_version",
'dlrnauth_username': '******',
'dlrnauth_password': '******',
'promotion_criteria_map': {
'current-tripleo': ['job1', 'job2']
},
'log_level': "INFO",
"log_file": "/dev/null"
}
file_config = {'main': {}}
conf_ok = pconfig.sanity_check(config, file_config)
self.assertTrue(conf_ok)
mock_log_warning.assert_has_calls([
mock.call(
'Missing parameter in configuration file: %s. '
'Using default value: %s', "distro_version",
"default_version"),
mock.call(
'Missing parameter in configuration file: %s. '
'Using default value: %s', "log_file", "/dev/null"),
mock.call(
'Missing parameter in configuration file: %s. '
'Using default value: %s', "distro_name", "default_distro"),
mock.call(
'Missing parameter in configuration file: username. '
'Using default value: %s', "ciuser")
],
any_order=True)
self.assertFalse(mock_log_error.called)
def test_load_defective_ini_file(self, mock_error, mock_warning):
os.environ["DLRNAPI_PASSWORD"] = "******"
ini_config = self.filepaths['missing_parameters']
with self.assertRaises(ConfigError):
PromoterLegacyConfig(ini_config)
calls = [
mock.call(
'Missing parameter in configuration file: %s. Using '
'default value: %s', 'distro_name', 'centos')
]
mock_warning.assert_has_calls(calls)
calls = [
mock.call('Invalid Log file: %s', '/dev/nul'),
mock.call('Error in configuration file %s' % str(ini_config))
]
mock_error.assert_has_calls(calls, any_order=True)
def setUp(self):
content = test_ini_configurations['correct']
fp, self.filepath = tempfile.mkstemp(prefix="instance_test")
with os.fdopen(fp, "w") as test_file:
test_file.write(content)
cli = "--config-file {} promote-all".format(self.filepath)
os.environ["DLRNAPI_PASSWORD"] = "******"
overrides = {
"default_qcow_server": "staging",
"log_level": "DEBUG",
}
overrides_obj = type("FakeArgs", (), overrides)
args = arg_parser(cmd_line=cli)
config = PromoterLegacyConfig(args.config_file,
overrides=overrides_obj)
self.promoter = Promoter(config)
def test_load_correct_verify_extended_params(self):
self.maxDiff = None
# Test for load correctness
os.environ["DLRNAPI_PASSWORD"] = "******"
config = PromoterLegacyConfig(self.filepaths['correct'])
self.assertEqual(config.target_registries_push, True)
promotion_criteria_map = {
"current-tripleo": {
"periodic-tripleo-centos-7-master-containers-build-push",
"periodic-tripleo-centos-7-master-standalone"
}
}
self.assertDictEqual(promotion_criteria_map,
config.promotion_criteria_map)
# This is tricky, here we verified that the code correctly
# converted this value to int In PromoterConfigBase we verify if
# this value has just been correctly loaded as str from ini config
self.assertEqual(config.latest_hashes_count, 10)
def test_ini_files():
"""
Check that ini files for configuring the promoter can be loaded as
correct configuration
:return:
"""
setup_logging("test_ini_file", logging.ERROR)
log = logging.getLogger("test_ini_file")
__, promoter_root = get_root_paths("test_ini_file")
errors = False
for root, __, files in os.walk(os.path.join(promoter_root, "config")):
for file in files:
if file.endswith('.ini'):
full_path = os.path.join(root, file)
try:
PromoterLegacyConfig(full_path, checks=['criteria'])
except Exception as config_except:
log.exception(config_except)
errors = True
if errors:
raise Exception("Some config files contain errors")
def staged_env(request):
"""
Fixture that runs the staging environment provisioner with parameters,
yield the stage_info file produced and cleans up after
It has two parameters by default, to test the interaction for single
pipeline and for integration pipeline
:return: yields the stage_info dict
"""
close_logging("promoter-staging")
close_logging("promoter")
log = logging.getLogger('promoter-staging')
setup_cmd_line = ""
teardown_cmd_line = ""
# We are going to call the main in the staging passing a composed command
# line, so we are testing also that the argument parsing is working
# correctly instead of passing configuration directly
release_config = \
"CentOS-7/master.yaml"
promoter_config_file = "staging/CentOS-7/master.ini"
setup_cmd_line += " --scenes dlrn"
try:
test_case = request.param
except AttributeError:
pass
except KeyError:
log.error("Invalid test case '{}'".format(request.param))
raise
# for the tests of the integration pipeline we need to pass a different
# file with db data
if "_integration" in test_case:
release_config = \
"CentOS-8/master.yaml"
promoter_config_file = \
"staging/CentOS-8/master.ini"
setup_cmd_line += " --db-data-file integration-pipeline.yaml"
teardown_cmd_line += " --db-data-file integration-pipeline.yaml"
setup_cmd_line += " setup --release-config {}".format(release_config)
teardown_cmd_line += " teardown"
log.info("Running cmd line: {}".format(setup_cmd_line))
config = stage_main(setup_cmd_line)
stage_info_path = config['stage_info_path']
with open(stage_info_path, "r") as stage_info_file:
stage_info = yaml.safe_load(stage_info_file)
overrides = {
'log_file': stage_info['main']['log_file'],
'repo_url': stage_info['dlrn']['server']['repo_url'],
'allowed_clients': 'dlrn_client',
'config_file': promoter_config_file,
}
overrides_obj = type("FakeArgs", (), overrides)
os.environ["DLRNAPI_PASSWORD"] = stage_info['dlrn']['server']['password']
if 'legacyconf' in test_case:
config = PromoterLegacyConfig(overrides_obj.config_file,
overrides=overrides_obj)
else:
raise Exception("New config engine is not implemented yet")
promoter = Promoter(config)
yield stage_info, promoter
log.info("Running cmd line: {}".format(teardown_cmd_line))
stage_main(teardown_cmd_line)
def test_expand_config_all_defaults(self, get_api_url_mock):
self.maxDiff = None
# All fields already present
api_url = "http://localhost:58080"
expected_config = {
'allowed_clients':
['registries_client', 'qcow_client', 'dlrn_client'],
'api_url':
api_url,
'containers_list_base_url':
'https://opendev.org/openstack/tripleo-common/raw/commit/',
'containers_list_path':
'container-images/tripleo_containers.yaml',
"containers_list_exclude_config":
("https://opendev.org/openstack/"
"tripleo-ci/raw/branch/master/roles/build-containers/vars"
"/main.yaml"),
'build_method':
'tripleo',
'container_preffix':
'openstack-',
'distro':
'centos7',
'distro_name':
'centos',
'distro_version':
'7',
'dlrnauth_password':
None,
'dlrnauth_username':
'******',
'dry_run':
False,
'latest_hashes_count':
10,
'log_file':
mock.ANY,
'log_level':
20,
'manifest_push':
False,
'promotion_criteria_map': {},
'release':
'master',
'repo_url':
'https://trunk.rdoproject.org/centos7-master',
'source_namespace':
"tripleomaster",
'target_namespace':
"tripleomaster",
'target_registries_push':
True,
'overcloud_images': {
'qcow_servers': {
'local': {}
}
},
'default_qcow_server':
'local',
'qcow_server': {},
}
in_config = {
'promotion_criteria_map': {},
'overcloud_images': {
'qcow_servers': {
"local": {}
}
},
'default_qcow_server': "local"
}
get_api_url_mock.return_value = api_url
config = PromoterLegacyConfig(self.filepaths['correct'],
filters=[],
checks=[])
out_config = config.expand_config(in_config)
self.assertEqual(out_config, expected_config)
def staged_env(request):
"""
Fixture that runs the staging environment provisioner with parameters,
yield the stage_info file produced and a promoter configured to use it
and cleans up after
It has two parameters by default, to test the interaction for single
pipeline and for integration pipeline
:param request: the parameter for the fixture, passed by the
pytest.mark.parametrize decorator above each test
:return: yields the stage_info dict and a promoter object
"""
# With a series of test ir rapid sequence but in the same test instance,
# logging configuration is passed from env to env, but log file won't be
# there anymore, so we need to close the logging handlers
close_logging("promoter-staging")
close_logging("promoter")
log = logging.getLogger('promoter-staging')
# We are going to call the main in the staging passing a composed command
# line, so we are testing also that the argument parsing is working
# correctly instead of passing configuration directly
# TODO (akahat) Need to enable legacy non integration pipeline coverage.
release_config = "CentOS-8/master.yaml"
promoter_config_file = \
"staging/CentOS-8/master.ini"
test_case = "all_legacyconf_integration"
try:
test_case = request.param
except AttributeError:
pass
except KeyError:
log.error("Invalid test case '{}'".format(request.param))
raise
# Select scenes to run in the staging env depending on the parameter passed
# to the fixture
extra_file = ""
scenes = None
if "all_" in test_case:
extra_file = "--extra-settings stage-config.yaml"
if "qcow_" in test_case:
scenes = 'dlrn,overcloud_images'
if "registries_" in test_case:
scenes = 'registries'
if "containers_" in test_case:
scenes = 'dlrn,registries,containers'
extra_file = "--extra-settings stage-config.yaml"
setup_cmd_line = " {}".format(extra_file)
teardown_cmd_line = "{}".format(extra_file)
if scenes is not None:
setup_cmd_line += " --scenes {}".format(scenes)
if "_integration" in test_case:
promoter_config_file = \
"staging/CentOS-8/master.ini"
setup_cmd_line += " --db-data-file integration-pipeline.yaml"
teardown_cmd_line += " --db-data-file integration-pipeline.yaml"
setup_cmd_line += " setup --release-config {}".format(release_config)
teardown_cmd_line += " teardown "
experimental = 'false'
if "_experimental" in test_case:
experimental = 'true'
# for the tests of the integration pipeline we need to pass a different
# file for component db data, and emulate CentOS8/master at least
log.info("Running cmd line: {}".format(setup_cmd_line))
config = stage_main(setup_cmd_line)
stage_info_path = config['stage_info_path']
with open(stage_info_path, "r") as stage_info_file:
stage_info = yaml.safe_load(stage_info_file)
overrides = {
'log_file': stage_info['main']['log_file'],
'repo_url': stage_info['dlrn']['server']['repo_url'],
'log_level': 'DEBUG',
'experimental': experimental,
'default_qcow_server': 'staging',
'config_file': promoter_config_file,
}
if "containers_" in test_case:
overrides['containers_list_base_url'] = \
stage_info['containers']['containers_list_base_url']
overrides_obj = type("FakeArgs", (), overrides)
os.environ["DLRNAPI_PASSWORD"] = stage_info['dlrn']['server']['password']
if 'legacyconf' in test_case:
config = PromoterLegacyConfig(overrides_obj.config_file,
overrides=overrides_obj)
else:
config_builder = PromoterConfigFactory()
config = config_builder("staging", release_config,
cli_args=overrides_obj)
promoter = Promoter(config)
yield stage_info, promoter
log.info("Running cmd line: {}".format(teardown_cmd_line))
stage_main(teardown_cmd_line)